diskio

// The activity of each disk of a Linux, by SSH: operations and throughput for reads and writes, the mean
// wait of an operation, and the share of time the disk was busy, over the interval between two runs.
// Reads /proc/diskstats with cat, and the list of whole disks with ls /sys/block: the kernel writes both
// the same way on Alpine, Debian and Fedora. /sys/block holds disks only, never their partitions: it is
// what leaves partitions out, whatever their name (sda1, nvme0n1p1, mmcblk0p1). Idle virtual disks (loop,
// ram, zram, optical drives, nbd) are left out by the default selection.
// The busy share only means something for a disk that serves one operation at a time: an SSD or NVMe at
// 100 % may still serve many more.
need('protocol:ssh');

per('disk', 'Disks', array('ignore' => array('loop*', 'ram*', 'zram*', 'sr*', 'fd*', 'nbd*')));
output('read-iops', 'number', array('label' => 'Reads', 'unit' => 'op/s', 'per' => 'disk'));
output('write-iops', 'number', array('label' => 'Writes', 'unit' => 'op/s', 'per' => 'disk'));
output('read-rate', 'number', array('label' => 'Read throughput', 'unit' => 'MiB/s', 'per' => 'disk'));
output('write-rate', 'number', array('label' => 'Write throughput', 'unit' => 'MiB/s', 'per' => 'disk'));
output('read-wait', 'number', array('label' => 'Mean read wait', 'unit' => 'ms', 'per' => 'disk'));
output('write-wait', 'number', array('label' => 'Mean write wait', 'unit' => 'ms', 'per' => 'disk'));
output('busy', 'number', array('label' => 'Busy time', 'unit' => '%', 'per' => 'disk'));

watch.above('read-latency', 'Slow reads', 'read-wait', array('warning' => 50, 'critical' => 200, 'for' => '5 min'));
watch.above('write-latency', 'Slow writes', 'write-wait', array('warning' => 50, 'critical' => 200, 'for' => '5 min'));
watch.above('busy', 'Disk saturated', 'busy', array('warning' => 90, 'for' => '10 min', 'enabled' => false));

// The whole disks. A name holding "/" (cciss/c0d0) is written "!" in /sys/block.
$block = ssh.run('ls /sys/block');
$disks = array();
foreach (text.words($block['output']) as $name) {
    $disks[text.join(text.split($name, '!'), '/')] = true;
}
if (list.count($disks) == 0) {
    fail('misconfigured', 'No disk found in /sys/block', $block['errors']);
}

// /proc/diskstats, one line per disk and partition: "8 0 sda", then reads completed, merged, sectors read,
// milliseconds spent reading; the same four for writes; operations in progress, milliseconds busy, and
// others not used here. A sector there is always 512 bytes.
$stats = ssh.run('cat /proc/diskstats');
$found = 0;
$measured = 0;
foreach (text.lines($stats['output']) as $line) {
    $f = text.words($line);
    if (list.count($f) < 14 || !list.has($disks, $f[2])) {
        continue;
    }
    $name = $f[2];
    $found = $found + 1;
    $reads = counter.rate('reads', $f[3], $name);
    $read_sectors = counter.rate('read-sectors', $f[5], $name);
    $read_ms = counter.rate('read-ms', $f[6], $name);
    $writes = counter.rate('writes', $f[7], $name);
    $write_sectors = counter.rate('write-sectors', $f[9], $name);
    $write_ms = counter.rate('write-ms', $f[10], $name);
    $busy_ms = counter.rate('busy-ms', $f[12], $name);
    if ($reads == '' || $read_sectors == '' || $read_ms == '' || $writes == '' || $write_sectors == '' || $write_ms == '' || $busy_ms == '') {
        continue;
    }
    $measured = $measured + 1;
    value('read-iops', number.round($reads, 1), $name);
    value('write-iops', number.round($writes, 1), $name);
    value('read-rate', number.round($read_sectors * 512 / 1048576, 2), $name);
    value('write-rate', number.round($write_sectors * 512 / 1048576, 2), $name);
    // The wait of an operation only exists if there was one.
    if ($reads > 0) {
        value('read-wait', number.round($read_ms / $reads, 1), $name);
    }
    if ($writes > 0) {
        value('write-wait', number.round($write_ms / $writes, 1), $name);
    }
    // Busy milliseconds per second: a thousand make 100 %.
    value('busy', number.round($busy_ms / 10, 1), $name);
}
if ($found == 0) {
    fail('misconfigured', 'No disk of /sys/block found in /proc/diskstats', $stats['errors']);
}
if ($measured == 0) {
    skip('First reading, or counters went backwards: activity comes with the next run');
}