pools

// The ZFS pools of a Linux or FreeBSD machine, by SSH: the health, space and fragmentation of each pool
// (zpool list), the data errors and device errors it has seen, and how long ago its last scrub finished
// (zpool status). It serves any machine with OpenZFS or the ZFS of FreeBSD: a TrueNAS, a Proxmox, a
// FreeBSD, a Linux with zfsutils.
// -H and -p make zpool write exact numbers, separated by tabs, with no header; zpool status -p writes its
// error counts exact too. The end of a scrub is written in the machine's local time: its age is counted
// against the machine's own clock, read in the same local time with date.
need('protocol:ssh');

per('pool', 'Pools');
output('health', 'text', array('label' => 'Health', 'per' => 'pool'));
output('used', 'number', array('label' => 'Used space', 'unit' => '%', 'per' => 'pool'));
output('free', 'number', array('label' => 'Free space', 'unit' => 'GiB', 'per' => 'pool'));
output('size', 'number', array('label' => 'Size', 'unit' => 'GiB', 'per' => 'pool'));
output('fragmentation', 'number', array('label' => 'Fragmentation', 'unit' => '%', 'per' => 'pool'));
output('data-errors', 'number', array('label' => 'Data errors', 'per' => 'pool'));
output('device-errors', 'number', array('label' => 'Device errors', 'per' => 'pool'));
output('scrub', 'text', array('label' => 'Last scrub', 'per' => 'pool'));
output('scrub-age', 'number', array('label' => 'Time since the last scrub', 'unit' => 'd', 'per' => 'pool'));

watch.expect('unhealthy', 'Pool not online', 'health', array('warning' => array('online')));
watch.state('failed', 'Pool failed', 'health', array('critical' => array('faulted', 'unavail', 'removed', 'suspended')));
watch.above('used', 'Used space', 'used', array('warning' => 80, 'critical' => 90));
watch.above('full-soon', 'Full within a week', 'used', array('warning' => 99, 'ahead' => '7 d'));
watch.above('fragmented', 'Pool fragmented', 'fragmentation', array('warning' => 70, 'enabled' => false));
watch.above('data-errors', 'Data errors', 'data-errors', array('critical' => 0));
watch.above('device-errors', 'Device errors', 'device-errors', array('warning' => 0));
watch.state('never-scrubbed', 'Pool never scrubbed', 'scrub', array('warning' => array('never')));
watch.above('scrub-old', 'Scrub too old', 'scrub-age', array('warning' => 35));
watch.missing('missing', 'Pool missing', 'health', array('enabled' => false));

// The seconds from 1 January 2000 to a date and time, all in the same local time: two of them subtract into
// a duration, off by an hour at most across a change of summer time.
function local_seconds($year, $month, $day, $hour, $minute, $second) {
    $before = array(0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334);
    $days = $before[$month - 1] + $day - 1;
    if ($month > 2 && $year % 4 == 0 && ($year % 100 != 0 || $year % 400 == 0)) {
        $days = $days + 1;
    }
    for ($y = 2000; $y < $year; $y = $y + 1) {
        $days = $days + 365;
        if ($y % 4 == 0 && ($y % 100 != 0 || $y % 400 == 0)) {
            $days = $days + 1;
        }
    }
    return $days * 86400 + $hour * 3600 + $minute * 60 + $second;
}

// One line per pool: "tank<TAB>3985729650688<TAB>1052266987520<TAB>2933462663168<TAB>26<TAB>3<TAB>ONLINE", the
// sizes in bytes; the fragmentation reads "-" on a pool that does not count it.
$list = ssh.run('zpool list -H -p -o name,size,alloc,free,cap,frag,health');
if ($list['exit'] != 0) {
    fail('misconfigured', 'zpool list failed', $list['errors']);
}
$pools = array();
foreach (text.lines($list['output']) as $line) {
    $w = text.words($line);
    if (list.count($w) < 7) {
        continue;
    }
    $name = $w[0];
    $pools[$name] = true;
    value('health', text.lower($w[6]), $name);
    if ($w[1] > 0) {
        value('used', number.round(100 * $w[2] / $w[1], 1), $name);
        value('free', number.round($w[3] / 1073741824, 1), $name);
        value('size', number.round($w[1] / 1073741824, 1), $name);
    }
    if (text.is_number($w[5])) {
        value('fragmentation', $w[5], $name);
    }
}
if (list.count($pools) == 0) {
    fail('misconfigured', 'No ZFS pool on this machine', $list['output'] . $list['errors']);
}

// zpool status -p, one section per pool:
//   pool: tank
//   state: ONLINE
//   scan: scrub repaired 0 in 00:12:31 with 0 errors on Sun Oct 11 00:36:32 2026
//   config:
//     NAME        STATE     READ WRITE CKSUM
//     tank        ONLINE       0     0     0
//       mirror-0  ONLINE       0     0     0
//         sda     ONLINE       0     0     0
//   errors: No known data errors
// The device errors are the sum of the three counts of every device of the pool.
$status = ssh.run('zpool status -p');
if ($status['exit'] != 0) {
    fail('misconfigured', 'zpool status failed', $status['errors']);
}
$months = array('Jan' => 1, 'Feb' => 2, 'Mar' => 3, 'Apr' => 4, 'May' => 5, 'Jun' => 6,
    'Jul' => 7, 'Aug' => 8, 'Sep' => 9, 'Oct' => 10, 'Nov' => 11, 'Dec' => 12);
$scrub = array();
$scrubbed_at = array();
$device_errors = array();
$data_errors = array();
$pool = '';
$in_config = false;
foreach (text.lines($status['output']) as $line) {
    $w = text.words($line);
    if ($w[0] == 'pool:' && list.count($w) > 1) {
        $pool = $w[1];
        $in_config = false;
        $scrub[$pool] = 'never';
        $device_errors[$pool] = 0;
        continue;
    }
    if ($pool == '') {
        continue;
    }
    if ($w[0] == 'scan:') {
        $m = text.match($line, 'scrub repaired .* on [A-Za-z]+ ([A-Za-z]+) +([0-9]+) ([0-9]+):([0-9]+):([0-9]+) ([0-9]+)$');
        if (list.count($m) > 0 && list.has($months, $m[1])) {
            $scrub[$pool] = 'finished';
            $scrubbed_at[$pool] = local_seconds($m[6], $months[$m[1]], $m[2], $m[3], $m[4], $m[5]);
        } elseif (text.contains($line, 'scrub in progress')) {
            $scrub[$pool] = 'running';
        } elseif (text.contains($line, 'scrub canceled')) {
            $scrub[$pool] = 'canceled';
        } elseif (text.contains($line, 'scrub')) {
            // A scrub on record in a form not read here: never say "never" of it.
            $scrub[$pool] = 'unreadable';
        }
        continue;
    }
    if ($w[0] == 'NAME' && list.count($w) >= 5 && $w[2] == 'READ') {
        $in_config = true;
        continue;
    }
    if ($w[0] == 'errors:') {
        $in_config = false;
        $data_errors[$pool] = 0;
        $m = text.match($line, '([0-9]+) data errors');
        if (list.count($m) > 0) {
            $data_errors[$pool] = $m[1];
        }
        continue;
    }
    if ($in_config && list.count($w) >= 5 && text.is_number($w[2]) && text.is_number($w[3]) && text.is_number($w[4])) {
        $device_errors[$pool] = $device_errors[$pool] + $w[2] + $w[3] + $w[4];
    }
}

// The machine's clock, in its local time: "2026 10 27 14 03 05".
$clock = ssh.run('date "+%Y %m %d %H %M %S"');
$c = text.words($clock['output']);
if (list.count($c) < 6) {
    fail('misconfigured', 'The clock of the machine could not be read', $clock['output'] . $clock['errors']);
}
$now = local_seconds($c[0], $c[1], $c[2], $c[3], $c[4], $c[5]);

foreach ($pools as $name => $seen) {
    if (!list.has($scrub, $name)) {
        continue;
    }
    value('scrub', $scrub[$name], $name);
    value('device-errors', $device_errors[$name], $name);
    if (list.has($data_errors, $name)) {
        value('data-errors', $data_errors[$name], $name);
    }
    if (list.has($scrubbed_at, $name) && $now >= $scrubbed_at[$name]) {
        value('scrub-age', number.round(($now - $scrubbed_at[$name]) / 86400, 1), $name);
    }
}