disks

// The SMART health of the disks of a Linux, by SSH: for each SATA, SAS or NVMe disk, its overall health, its
// temperature, its reallocated, pending and uncorrectable sectors, its power-on hours and, for an SSD or an
// NVMe, its remaining life. A disk is named by its serial number, which follows it from one slot or one boot
// to the next; its path (/dev/sda) is an output.
// smartctl sends raw commands to the disk, which the kernel allows to root only: it runs through sudo, with
// -n so that sudo never waits for a password, and only in the forms the sudoers line of the documentation
// allows (a fixed path, fixed options, a disk named sdX, sdXY, nvmeN or nvmeNN). -j makes smartctl answer in
// JSON (smartmontools 7.0 and later); -n standby makes it leave a spun-down disk asleep. A disk asleep keeps
// the health and sectors last read from it, so that a failure never goes out because the disk sleeps.
need('protocol:ssh');

per('disk', 'Disks');
output('health', 'text', array('label' => 'SMART health', 'per' => 'disk'));
output('device', 'text', array('label' => 'Device', 'per' => 'disk'));
output('power', 'text', array('label' => 'Power state', 'per' => 'disk'));
output('read-age', 'number', array('label' => 'Age of the SMART reading', 'unit' => 'h', 'per' => 'disk'));
output('temperature', 'number', array('label' => 'Temperature', 'unit' => '°C', 'per' => 'disk'));
output('reallocated', 'number', array('label' => 'Reallocated sectors', 'per' => 'disk'));
output('pending', 'number', array('label' => 'Pending sectors', 'per' => 'disk'));
output('uncorrectable', 'number', array('label' => 'Uncorrectable sectors', 'per' => 'disk'));
output('power-on', 'number', array('label' => 'Power-on time', 'unit' => 'h', 'per' => 'disk'));
output('remaining-life', 'number', array('label' => 'Remaining life', 'unit' => '%', 'per' => 'disk'));

watch.state('failed', 'SMART health failed', 'health', array('critical' => array('failed')));
watch.state('unknown', 'SMART health unknown', 'health', array('warning' => array('unknown')));
// The health of a disk asleep is that of its last reading, which ages. A week: a disk asleep a night or a few
// days stays green; one not read for more than a week is shown with a health nobody has checked since.
watch.above('stale', 'SMART reading old', 'read-age', array('warning' => 168));
watch.above('hot', 'High temperature', 'temperature', array('warning' => 55, 'critical' => 60, 'for' => '3 runs'));
watch.increase('reallocated', 'New reallocated sectors', 'reallocated', array('warning' => 0, 'window' => '7 d'));
watch.above('pending', 'Pending sectors', 'pending', array('warning' => 0));
watch.increase('uncorrectable', 'New uncorrectable sectors', 'uncorrectable', array('critical' => 0, 'window' => '7 d'));
watch.below('worn', 'Disk worn out', 'remaining-life', array('warning' => 10, 'critical' => 3));
watch.missing('missing', 'Disk missing', 'health', array('enabled' => false));

// What sudo or smartctl said, when the answer is not JSON: sudo refusing the command (no sudoers line, a line
// that does not match, a password asked), smartctl missing, or a smartctl older than 7.0, which knows no -j.
function not_json($read) {
    if (text.contains($read['errors'], 'sudo')) {
        fail('misconfigured', 'sudo refused to run smartctl: see the sudoers line of the documentation', $read['errors']);
    }
    fail('misconfigured', 'smartctl did not answer in JSON (smartmontools 7.0 or later is needed)', $read['output'] . $read['errors']);
}

// The disks smartctl can open: {"devices": [{"name": "/dev/sda", "type": "sat", "protocol": "ATA"}, ...]}.
// Its exit status is a set of bits: 1, the command line was not understood.
$scan = ssh.run('sudo -n /usr/sbin/smartctl --scan-open -j');
$answer = text.trim($scan['output']);
if (list.count(text.match($answer, '^\{')) == 0) {
    not_json($scan);
}
$scanned = json.read($answer);
if ($scan['exit'] % 2 == 1 || !list.has($scanned, 'devices')) {
    fail('misconfigured', 'smartctl did not list the disks', $answer . $scan['errors']);
}

// Only the disks the sudoers line allows are read: /dev/sda, /dev/sdab, /dev/nvme0, /dev/nvme12. A disk behind
// a RAID controller (/dev/bus/0 with -d megaraid,N) or any other name is left out.
$disks = array();
foreach ($scanned['devices'] as $device) {
    $m = text.match(list.get($device, 'name', ''), '^/dev/(sd[a-z]{1,2}|nvme[0-9]{1,2})$');
    if (list.count($m) > 0) {
        $disks[$m[0]] = $m[1];
    }
}
if (list.count($disks) == 0) {
    fail('misconfigured', 'smartctl found no SATA, SAS or NVMe disk', $answer);
}

// Returns what was read from a disk, the last time it was awake: its health, its sectors and its remaining
// life, each when the disk serves it.
function last_reading($name, $reading) {
    value('health', $reading['health'], $name);
    if (list.has($reading, 'reallocated')) {
        value('reallocated', $reading['reallocated'], $name);
    }
    if (list.has($reading, 'pending')) {
        value('pending', $reading['pending'], $name);
    }
    if (list.has($reading, 'uncorrectable')) {
        value('uncorrectable', $reading['uncorrectable'], $name);
    }
    if (list.has($reading, 'remaining-life')) {
        value('remaining-life', $reading['remaining-life'], $name);
    }
}

// A disk asleep tells nothing, its serial number included. The last reading of each disk awake is kept by
// path: its serial number names the disk asleep, and its health and sectors stand for it, with their time.
// A disk never seen awake, or a disk that serves no serial number, is named by the kernel's name (sda).
$known = array();
if (cache.has('readings')) {
    $known = cache.get('readings');
}
$readings = array();
$now = time.now();
// A disk seen through two paths (multipath) gives the same serial number twice: it is returned once.
$named = array();

// The SMART attributes of an ATA disk that give its remaining life in their normalized value, from 100 when
// new down to 0, under the names smartmontools gives them: Samsung, Kingston and SandForce, Intel, Crucial
// and Micron.
$life_attributes = array('Wear_Leveling_Count' => true, 'SSD_Life_Left' => true,
    'Media_Wearout_Indicator' => true, 'Percent_Lifetime_Remain' => true);

foreach ($disks as $path => $kernel_name) {
    $read = ssh.run('sudo -n /usr/sbin/smartctl -j -n standby -a', arguments=array($path));
    $text = text.trim($read['output']);
    if (list.count(text.match($text, '^\{')) == 0) {
        not_json($read);
    }
    $disk = json.read($text);
    // Exit bit 1: the command line was not understood.
    if ($read['exit'] % 2 == 1) {
        fail('misconfigured', 'smartctl did not understand its command line', $text . $read['errors']);
    }

    // Exit bit 2: the disk was not read. Either it is asleep, which smartctl says ("Device is in STANDBY
    // mode", or SLEEP), or it could not be opened (pulled out since the scan, a bridge that does not pass
    // SMART through). The other bits tell the disk's state, which the JSON says too.
    if ($read['exit'] % 4 >= 2) {
        $last = list.get($known, $path, array());
        $name = list.get($last, 'serial', $kernel_name);
        if (list.has($named, $name)) {
            continue;
        }
        $named[$name] = true;
        value('device', $path, $name);
        if (!text.contains($text, 'STANDBY mode') && !text.contains($text, 'SLEEP mode')) {
            value('health', 'unknown', $name);
            continue;
        }
        value('power', 'standby', $name);
        if (list.count($last) == 0) {
            value('health', 'standby', $name);
            continue;
        }
        $readings[$path] = $last;
        last_reading($name, $last);
        value('read-age', number.round(($now - $last['at']) / 3600, 1), $name);
        continue;
    }

    $serial = text.trim(list.get($disk, 'serial_number', ''));
    $name = $kernel_name;
    if ($serial != '') {
        $name = $serial;
    }
    if (list.has($named, $name)) {
        continue;
    }
    $named[$name] = true;
    value('device', $path, $name);
    value('power', 'active', $name);
    value('read-age', 0, $name);

    // smart_status.passed is false when the disk says it is failing: an ATA attribute at or under its
    // threshold, an NVMe critical warning, a SCSI failure prediction. Absent when the disk serves no SMART.
    $reading = array('at' => $now, 'health' => 'unknown');
    if ($serial != '') {
        $reading['serial'] = $serial;
    }
    $status = list.get($disk, 'smart_status', array());
    if (list.has($status, 'passed')) {
        if ($status['passed']) {
            $reading['health'] = 'passed';
        } else {
            $reading['health'] = 'failed';
        }
    }

    $temperature = list.get($disk, 'temperature', array());
    if (list.has($temperature, 'current')) {
        value('temperature', $temperature['current'], $name);
    }
    $power_on = list.get($disk, 'power_on_time', array());
    if (list.has($power_on, 'hours')) {
        value('power-on', $power_on['hours'], $name);
    }

    // An ATA disk: attributes 5 (reallocated), 197 (pending) and 198 (uncorrectable), by their raw value.
    $attributes = list.get($disk, 'ata_smart_attributes', array());
    foreach (list.get($attributes, 'table', array()) as $attribute) {
        $id = $attribute['id'];
        if ($id == 5) {
            $reading['reallocated'] = $attribute['raw']['value'];
        } elseif ($id == 197) {
            $reading['pending'] = $attribute['raw']['value'];
        } elseif ($id == 198) {
            $reading['uncorrectable'] = $attribute['raw']['value'];
        } elseif (list.has($life_attributes, list.get($attribute, 'name', ''))) {
            $reading['remaining-life'] = $attribute['value'];
        }
    }

    // An NVMe disk: the media errors are the data it could not read back; the percentage used may go past 100.
    $nvme = list.get($disk, 'nvme_smart_health_information_log', array());
    if (list.has($nvme, 'media_errors')) {
        $reading['uncorrectable'] = $nvme['media_errors'];
    }
    if (list.has($nvme, 'percentage_used')) {
        $life = 100 - $nvme['percentage_used'];
        if ($life < 0) {
            $life = 0;
        }
        $reading['remaining-life'] = $life;
    }

    // A SAS disk: the grown defect list counts the sectors reallocated since it left the factory; the read
    // errors the disk could not correct; the endurance used of an SSD.
    if (list.has($disk, 'scsi_grown_defect_list')) {
        $reading['reallocated'] = $disk['scsi_grown_defect_list'];
    }
    $errors = list.get($disk, 'scsi_error_counter_log', array());
    $read_errors = list.get($errors, 'read', array());
    if (list.has($read_errors, 'total_uncorrected_errors')) {
        $reading['uncorrectable'] = $read_errors['total_uncorrected_errors'];
    }
    if (list.has($disk, 'scsi_percentage_used_endurance_indicator')) {
        $life = 100 - $disk['scsi_percentage_used_endurance_indicator'];
        if ($life < 0) {
            $life = 0;
        }
        $reading['remaining-life'] = $life;
    }

    $readings[$path] = $reading;
    last_reading($name, $reading);
}

// Rewritten at every run, for the disks present only, and kept seven days, the longest the cache allows: a
// reading of a disk asleep is kept as long as the probe runs.
cache.set('readings', $readings, 604800);