gpus

// The NVIDIA GPUs of a Linux machine, by SSH, through nvidia-smi: for each GPU, its utilization, its memory,
// its temperature, its power draw, whether its hardware slows its clocks down (overheating, a power brake),
// and the uncorrected ECC errors of a data-center GPU. A GPU is named by its UUID, which follows the card
// from one slot or one boot to the next; its index and its model are outputs. nvidia-smi needs no root.
need('protocol:ssh');

per('gpu', 'GPUs');
output('index', 'text', array('label' => 'Index', 'per' => 'gpu'));
output('model', 'text', array('label' => 'Model', 'per' => 'gpu'));
output('utilization', 'number', array('label' => 'Utilization', 'unit' => '%', 'per' => 'gpu'));
output('memory-used', 'number', array('label' => 'Used memory', 'unit' => 'MiB', 'per' => 'gpu'));
output('memory-used-percent', 'number', array('label' => 'Used memory', 'unit' => '%', 'per' => 'gpu'));
output('temperature', 'number', array('label' => 'Temperature', 'unit' => '°C', 'per' => 'gpu'));
output('power', 'number', array('label' => 'Power draw', 'unit' => 'W', 'per' => 'gpu'));
output('slowdown', 'text', array('label' => 'Hardware slowdown', 'per' => 'gpu'));
output('ecc-errors', 'number', array('label' => 'Uncorrected ECC errors', 'per' => 'gpu'));

watch.above('hot', 'High temperature', 'temperature', array('warning' => 80, 'critical' => 90, 'for' => '3 runs'));
watch.state('slowdown', 'Hardware slowdown', 'slowdown', array('warning' => array('active'), 'for' => '3 runs'));
watch.above('ecc-errors', 'Uncorrected ECC errors', 'ecc-errors', array('critical' => 0));
watch.above('memory-full', 'GPU memory full', 'memory-used-percent', array('warning' => 95, 'for' => '3 runs', 'enabled' => false));
watch.missing('missing', 'GPU missing', 'model', array('enabled' => false));

// The name of the slowdown field changed with the drivers: clocks_throttle_reasons.hw_slowdown, then
// clocks_event_reasons.hw_slowdown. The list of the fields the driver knows tells which one to ask for: a
// field it does not know would make nvidia-smi refuse the whole query.
$help = ssh.run('nvidia-smi --help-query-gpu');
if ($help['exit'] != 0) {
    fail('misconfigured', 'nvidia-smi did not answer: no NVIDIA driver, or nvidia-smi not in the path',
        $help['output'] . $help['errors']);
}
$slowdown_field = 'clocks_throttle_reasons.hw_slowdown';
if (text.contains($help['output'], 'clocks_event_reasons.hw_slowdown')) {
    $slowdown_field = 'clocks_event_reasons.hw_slowdown';
}

// One line per GPU, its fields in the order asked, separated by ", ", without units:
// 0, GPU-5e4c..., NVIDIA A100-SXM4-40GB, 37, 10240, 40960, 54, 212.41, Not Active, 0
// A field the GPU does not serve reads [N/A] or [Not Supported] (ECC on a consumer GPU, power on some).
$read = ssh.run('nvidia-smi --query-gpu=index,uuid,name,utilization.gpu,memory.used,memory.total,temperature.gpu,power.draw,' . $slowdown_field . ',ecc.errors.uncorrected.volatile.total --format=csv,noheader,nounits');
if ($read['exit'] != 0) {
    fail('misconfigured', 'nvidia-smi refused the query', $read['output'] . $read['errors']);
}

$gpus = 0;
foreach (text.lines($read['output']) as $line) {
    $fields = text.split($line, ',');
    if (list.count($fields) != 10) {
        continue;
    }
    $f = array();
    foreach ($fields as $field) {
        $f = list.add($f, text.trim($field));
    }
    $name = $f[1];
    $gpus = $gpus + 1;
    value('index', $f[0], $name);
    value('model', $f[2], $name);
    if (text.is_number($f[3])) {
        value('utilization', $f[3], $name);
    }
    if (text.is_number($f[4])) {
        value('memory-used', $f[4], $name);
        if (text.is_number($f[5]) && $f[5] > 0) {
            value('memory-used-percent', number.round(100 * $f[4] / $f[5], 1), $name);
        }
    }
    if (text.is_number($f[6])) {
        value('temperature', $f[6], $name);
    }
    if (text.is_number($f[7])) {
        value('power', $f[7], $name);
    }
    if ($f[8] == 'Active') {
        value('slowdown', 'active', $name);
    } elseif ($f[8] == 'Not Active') {
        value('slowdown', 'not-active', $name);
    }
    if (text.is_number($f[9])) {
        value('ecc-errors', $f[9], $name);
    }
}
if ($gpus == 0) {
    fail('misconfigured', 'nvidia-smi lists no GPU', $read['output']);
}