s3-bucket-size

// The size of an S3 bucket and its number of objects, by the daily storage metrics Amazon S3 sends to CloudWatch,
// with an access key: the bytes of every storage class added up, the objects, and how old these figures are.
// Reads GET https://monitoring.<region>.amazonaws.com/?Action=ListMetrics, then ?Action=GetMetricStatistics once
// per storage class and once for the objects, signed by Signature Version 4.
need('protocol:http');
need('protocol:aws');
$region = variable('aws:region', 'text', array(
    'label' => 'Region',
    'help'  => 'The AWS region of the bucket, as AWS writes it in an API: eu-west-3, us-east-1.',
));
$bucket = variable('aws:bucket', 'text', array(
    'label' => 'Bucket',
    'help'  => 'The name of the bucket.',
));

output('size', 'number', array('label' => 'Size', 'unit' => 'GiB'));
output('objects', 'number', array('label' => 'Objects'));
output('age', 'number', array('label' => 'Age of the figures', 'unit' => 'h'));

watch.above('large', 'Bucket too large', 'size', array(
    'help' => 'Raises when the size goes above the thresholds set on the probe, in GiB.',
));
watch.above('many-objects', 'Too many objects', 'objects', array(
    'help'    => 'Raises when the number of objects goes above the thresholds set on the probe.',
    'enabled' => false,
));
watch.above('stale', 'Figures not fresh', 'age', array('warning' => 72));

// One signed GET on an AWS API of the Query protocol (EC2, RDS, CloudWatch): the parameters in the address, the
// answer in XML. aws.sign gives the headers that carry the signature: the keys never leave. A refusal is told
// by the error code AWS writes in its answer; a throttled request or a fault of AWS is not one.
function aws_get($service, $region, $query, $what) {
    $url = 'https://' . $service . '.' . $region . '.amazonaws.com/?' . url.query($query);
    $answer = http.get($url, headers=aws.sign('GET', $url, array(), '', $region, $service));
    if ($answer['status'] != 200) {
        $code = '';
        $found = text.match($answer['body'], '<Code>([^<]+)</Code>');
        if (list.count($found) > 0) {
            $code = $found[1];
        }
        if ($code == 'Throttling' || $code == 'RequestLimitExceeded' || $answer['status'] >= 500) {
            fail('unreachable', 'AWS answers the reading of ' . $what . ' with code ' . $answer['status'] . ' ' . $code);
        }
        if ($answer['status'] == 400 || $answer['status'] == 401 || $answer['status'] == 403) {
            fail('misconfigured', 'AWS refuses to read ' . $what . ', code ' . $answer['status'] . ' ' . $code,
                'Check the access key, the permissions of its user or role, and the region.');
        }
        fail('unreachable', 'AWS answers the reading of ' . $what . ' with code ' . $answer['status']);
    }
    return xml.read($answer['body']);
}

// The first child of that name, array() when it is absent or empty. An empty element, such as a list without
// members written <name/>, reads as the text '' and not as an array.
function element($parent, $name) {
    $children = list.get($parent, $name, array());
    if (list.count($children) == 0 || !list.is($children[0])) {
        return array();
    }
    return $children[0];
}

// The text of the first child of that name, '' when it is absent.
function child_text($parent, $name) {
    $children = list.get($parent, $name, array());
    if (list.count($children) == 0) {
        return '';
    }
    return $children[0];
}

// The newest daily average of one storage metric of the bucket over the last three days, as
// array('value' => …, 'at' => seconds since 1970), or array() when CloudWatch has none. S3 sends one point a
// day; CloudWatch gives the points in no order.
function newest_daily($region, $bucket, $metric, $storage, $now) {
    $query = array(
        'Action'                     => 'GetMetricStatistics',
        'Version'                    => '2010-08-01',
        'Namespace'                  => 'AWS/S3',
        'MetricName'                 => $metric,
        'Dimensions.member.1.Name'   => 'BucketName',
        'Dimensions.member.1.Value'  => $bucket,
        'Dimensions.member.2.Name'   => 'StorageType',
        'Dimensions.member.2.Value'  => $storage,
        'StartTime'                  => time.format($now - 3 * 86400, 'rfc3339'),
        'EndTime'                    => time.format($now, 'rfc3339'),
        'Period'                     => 86400,
        'Statistics.member.1'        => 'Average',
    );
    $result = element(element(aws_get('monitoring', $region, $query, 'the metric ' . $metric), 'GetMetricStatisticsResponse'),
        'GetMetricStatisticsResult');
    $newest = array();
    foreach (list.get(element($result, 'Datapoints'), 'member', array()) as $point) {
        $at = time.parse(child_text($point, 'Timestamp'), 'rfc3339');
        if (list.count($newest) == 0 || $at > $newest['at']) {
            $newest = array('value' => child_text($point, 'Average'), 'at' => $at);
        }
    }
    return $newest;
}

// The region goes into the name of the host: it must be a name.
if (list.count(text.match($region, '^[a-z0-9-]+$')) == 0) {
    fail('misconfigured', 'The region holds other characters than lower-case letters, digits and hyphens');
}

// BucketSizeBytes is sent once per storage class the bucket holds (StandardStorage, StandardIAStorage,
// GlacierStorage…): ListMetrics gives those of this bucket. <ListMetricsResponse><ListMetricsResult><Metrics>
// <member><Dimensions><member><Name>BucketName</Name><Value>…</Value></member><member><Name>StorageType</Name>
// <Value>StandardStorage</Value></member></Dimensions><MetricName>BucketSizeBytes</MetricName>…</member>
// </Metrics><NextToken>…</NextToken></ListMetricsResult>…</ListMetricsResponse>.
$storages = array();
$token = '';
while (true) {
    $query = array('Action' => 'ListMetrics', 'Version' => '2010-08-01', 'Namespace' => 'AWS/S3',
        'MetricName' => 'BucketSizeBytes', 'Dimensions.member.1.Name' => 'BucketName', 'Dimensions.member.1.Value' => $bucket);
    if ($token != '') {
        $query['NextToken'] = $token;
    }
    $result = element(element(aws_get('monitoring', $region, $query, 'the metrics of the bucket'), 'ListMetricsResponse'),
        'ListMetricsResult');
    foreach (list.get(element($result, 'Metrics'), 'member', array()) as $metric) {
        foreach (list.get(element($metric, 'Dimensions'), 'member', array()) as $dimension) {
            if (child_text($dimension, 'Name') == 'StorageType') {
                $storages = list.add($storages, child_text($dimension, 'Value'));
            }
        }
    }
    $token = child_text($result, 'NextToken');
    if ($token == '') {
        break;
    }
}
if (list.count($storages) == 0) {
    fail('misconfigured', 'CloudWatch has no size of the bucket ' . $bucket . ' in ' . $region,
        'Check the name and the region of the bucket: S3 sends its daily metrics in the region of the bucket.');
}

$now = number.round(time.now());
$objects = newest_daily($region, $bucket, 'NumberOfObjects', 'AllStorageTypes', $now);
if (list.count($objects) == 0) {
    fail('unreachable', 'CloudWatch has no number of objects of the bucket ' . $bucket . ' in the last three days');
}
$bytes = 0;
foreach ($storages as $storage) {
    $size = newest_daily($region, $bucket, 'BucketSizeBytes', $storage, $now);
    if (list.count($size) > 0) {
        $bytes = $bytes + $size['value'];
    }
}
value('size', number.round($bytes / 1073741824, 3));
value('objects', $objects['value']);
value('age', number.round(($now - $objects['at']) / 3600, 1));