pools

// The ZFS pools of a Linux or FreeBSD machine, by SSH: the health, space and fragmentation of each pool
// (zpool list), the data errors and device errors it has seen, and how long ago its last scrub finished
// (zpool status). It serves any machine with OpenZFS or the ZFS of FreeBSD: a TrueNAS, a Proxmox, a
// FreeBSD, a Linux with zfsutils.
// -H and -p make zpool write exact numbers, separated by tabs, with no header; zpool status -p writes its
// error counts exact too. The end of a scrub is written in the machine's local time, without its offset: the
// offset of the machine now is put on it, and its age is counted against the machine's own clock, read with
// date.
need('protocol:ssh');

per('pool', 'Pools');
output('health', 'text', array('label' => 'Health', 'per' => 'pool'));
output('used-percent', 'number', array('label' => 'Used space', 'unit' => '%', 'per' => 'pool'));
output('free', 'number', array('label' => 'Free space', 'unit' => 'GiB', 'per' => 'pool'));
output('size', 'number', array('label' => 'Size', 'unit' => 'GiB', 'per' => 'pool'));
output('fragmentation', 'number', array('label' => 'Fragmentation', 'unit' => '%', 'per' => 'pool'));
output('data-errors', 'number', array('label' => 'Data errors', 'per' => 'pool'));
output('device-errors', 'number', array('label' => 'Device errors', 'per' => 'pool'));
output('scrub', 'text', array('label' => 'Last scrub', 'per' => 'pool'));
output('scrub-age', 'number', array('label' => 'Time since the last scrub', 'unit' => 'd', 'per' => 'pool'));

watch.expect('unhealthy', 'Pool not online', 'health', array('warning' => array('online')));
watch.state('failed', 'Pool failed', 'health', array('critical' => array('faulted', 'unavail', 'removed', 'suspended')));
watch.above('used', 'Used space', 'used-percent', array('warning' => 80, 'critical' => 90));
watch.above('full-soon', 'Full within a week', 'used-percent', array('warning' => 99, 'ahead' => '7 d'));
watch.above('fragmented', 'Pool fragmented', 'fragmentation', array('warning' => 70, 'enabled' => false));
watch.above('data-errors', 'Data errors', 'data-errors', array('critical' => 0));
watch.above('device-errors', 'Device errors', 'device-errors', array('warning' => 0));
watch.state('never-scrubbed', 'Pool never scrubbed', 'scrub', array('warning' => array('never')));
watch.above('scrub-old', 'Scrub too old', 'scrub-age', array('warning' => 35));
watch.missing('missing', 'Pool missing', 'health', array('enabled' => false));

// One line per pool: "tank<TAB>3985729650688<TAB>1052266987520<TAB>2933462663168<TAB>26<TAB>3<TAB>ONLINE", the
// sizes in bytes; the fragmentation reads "-" on a pool that does not count it.
$list = ssh.run('zpool list -H -p -o name,size,alloc,free,cap,frag,health');
if ($list['exit'] != 0) {
    fail('misconfigured', 'zpool list failed', $list['errors']);
}
$pools = array();
foreach (text.lines($list['output']) as $line) {
    $w = text.words($line);
    if (list.count($w) < 7) {
        continue;
    }
    $name = $w[0];
    $pools[$name] = true;
    value('health', text.lower($w[6]), $name);
    if ($w[1] > 0) {
        value('used-percent', number.round(100 * $w[2] / $w[1], 1), $name);
        value('free', number.round($w[3] / 1073741824, 1), $name);
        value('size', number.round($w[1] / 1073741824, 1), $name);
    }
    if (text.is_number($w[5])) {
        value('fragmentation', $w[5], $name);
    }
}
if (list.count($pools) == 0) {
    fail('misconfigured', 'No ZFS pool on this machine', $list['output'] . $list['errors']);
}

// zpool status -p, one section per pool:
//   pool: tank
//   state: ONLINE
//   scan: scrub repaired 0 in 00:12:31 with 0 errors on Sun Oct 11 00:36:32 2026
//   config:
//     NAME        STATE     READ WRITE CKSUM
//     tank        ONLINE       0     0     0
//       mirror-0  ONLINE       0     0     0
//         sda     ONLINE       0     0     0
//   errors: No known data errors
// The device errors are the sum of the three counts of every device of the pool.
$status = ssh.run('zpool status -p');
if ($status['exit'] != 0) {
    fail('misconfigured', 'zpool status failed', $status['errors']);
}
// The end of a scrub is kept rewritten 2026-10-11T00:36:32, still in the machine's local time.
$months = array('Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
    'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12');
$scrub = array();
$scrubbed_at = array();
$device_errors = array();
$data_errors = array();
$pool = '';
$in_config = false;
foreach (text.lines($status['output']) as $line) {
    $w = text.words($line);
    if ($w[0] == 'pool:' && list.count($w) > 1) {
        $pool = $w[1];
        $in_config = false;
        $scrub[$pool] = 'never';
        $device_errors[$pool] = 0;
        continue;
    }
    if ($pool == '') {
        continue;
    }
    if ($w[0] == 'scan:') {
        $m = text.match($line, 'scrub repaired .* on [A-Za-z]+ ([A-Za-z]+) +([0-9]+) ([0-9]+:[0-9]+:[0-9]+) ([0-9]+)$');
        if (list.count($m) > 0 && list.has($months, $m[1])) {
            $scrub[$pool] = 'finished';
            $day = $m[2];
            if (text.length($day) == 1) {
                $day = '0' . $day;
            }
            $scrubbed_at[$pool] = $m[4] . '-' . $months[$m[1]] . '-' . $day . 'T' . $m[3];
        } elseif (text.contains($line, 'scrub in progress')) {
            $scrub[$pool] = 'running';
        } elseif (text.contains($line, 'scrub canceled')) {
            $scrub[$pool] = 'canceled';
        } elseif (text.contains($line, 'scrub')) {
            // A scrub on record in a form not read here: never say "never" of it.
            $scrub[$pool] = 'unreadable';
        }
        continue;
    }
    if ($w[0] == 'NAME' && list.count($w) >= 5 && $w[2] == 'READ') {
        $in_config = true;
        continue;
    }
    if ($w[0] == 'errors:') {
        $in_config = false;
        $data_errors[$pool] = 0;
        $m = text.match($line, '([0-9]+) data errors');
        if (list.count($m) > 0) {
            $data_errors[$pool] = $m[1];
        }
        continue;
    }
    if ($in_config && list.count($w) >= 5 && text.is_number($w[2]) && text.is_number($w[3]) && text.is_number($w[4])) {
        $device_errors[$pool] = $device_errors[$pool] + $w[2] + $w[3] + $w[4];
    }
}

// The machine's clock, in seconds since 1970, and the offset of its local time now: "1793107815 +0100". The
// offset is put on the end of each scrub.
$clock = ssh.run('date "+%s %z"');
$c = text.words($clock['output']);
if (list.count($c) < 2 || !text.is_number($c[0]) || list.count(text.match($c[1], '^[+-][0-9]{4}$')) == 0) {
    fail('misconfigured', 'The clock of the machine could not be read', $clock['output'] . $clock['errors']);
}
$now = $c[0] + 0;
$offset = $c[1];

foreach ($pools as $name => $seen) {
    if (!list.has($scrub, $name)) {
        continue;
    }
    value('scrub', $scrub[$name], $name);
    value('device-errors', $device_errors[$name], $name);
    if (list.has($data_errors, $name)) {
        value('data-errors', $data_errors[$name], $name);
    }
    if (list.has($scrubbed_at, $name)) {
        $ended = time.parse($scrubbed_at[$name] . $offset, 'iso8601');
        if ($now >= $ended) {
            value('scrub-age', number.round(($now - $ended) / 86400, 1), $name);
        }
    }
}