cpu

// CPU usage of a Linux, by SSH: each core and the whole, the share spent waiting for I/O (iowait) and
// the time stolen by the hypervisor (steal), over the interval between two runs.
// Reads /proc/stat with grep: the kernel writes it the same way on Alpine, Debian and Fedora, and grep
// without options is the same in BusyBox and GNU grep.
need('protocol:ssh');

per('core', 'Cores');
output('cpu-busy', 'number', array('label' => 'Overall usage', 'unit' => '%'));
output('iowait', 'number', array('label' => 'Waiting for I/O', 'unit' => '%'));
output('steal', 'number', array('label' => 'Stolen by the hypervisor', 'unit' => '%'));
output('core-busy', 'number', array('label' => 'Core usage', 'unit' => '%', 'per' => 'core'));
output('cores', 'number', array('label' => 'Number of cores'));

watch.above('cpu-high', 'CPU busy', 'cpu-busy', array('warning' => 85, 'critical' => 95, 'for' => '3 runs'));
watch.above('iowait-high', 'Waiting for disks', 'iowait', array('warning' => 20, 'for' => '5 min'));
watch.above('steal-high', 'Stolen by the hypervisor', 'steal', array('warning' => 10, 'for' => '5 min'));
watch.above('core-hot', 'A core saturated', 'core-busy', array('warning' => 95, 'for' => '3 runs', 'enabled' => false));
watch.below('core-idle', 'A core underused', 'core-busy', array('warning' => 20, 'enabled' => false));
watch.deviate('core-neglected', 'A core left idle', 'core-busy', array('side' => 'below', 'warning' => 30, 'enabled' => false));
watch.expect('core-count', 'Unexpected number of cores', 'cores');

// A "cpu" line for the whole, then "cpu0", "cpu1"... for each core, each followed by
// user nice system idle iowait irq softirq steal, in hundredths of a second since boot
// (guest and guest_nice, which follow, are already counted in user and nice).
$stat = ssh.run('grep ^cpu /proc/stat');
$global = '';
$iowait = '';
$steal = '';
$cores = 0;
foreach (text.lines($stat['output']) as $line) {
    $f = text.words($line);
    if (list.count($f) < 9) {
        fail('misconfigured', '/proc/stat has no steal column: kernel older than 2.6.11', $line);
    }
    $idle = counter.rate('idle', $f[4] + $f[5], $f[0]);
    $total = counter.rate('total', $f[1] + $f[2] + $f[3] + $f[4] + $f[5] + $f[6] + $f[7] + $f[8], $f[0]);
    $busy = '';
    if ($idle != '' && $total != '' && $total > 0) {
        $busy = number.round(100 * (1 - $idle / $total), 1);
    }
    if ($f[0] == 'cpu') {
        $global = $busy;
        // iowait may go backwards on some kernels: its rate is then unknown for this run.
        $wait = counter.rate('iowait', $f[5]);
        $stolen = counter.rate('steal', $f[8]);
        if ($busy != '' && $wait != '' && $stolen != '') {
            $iowait = number.round(100 * $wait / $total, 1);
            $steal = number.round(100 * $stolen / $total, 1);
        }
    } else {
        $cores = $cores + 1;
        if ($busy != '') {
            value('core-busy', $busy, $f[0]);
        }
    }
}
if ($cores == 0) {
    fail('misconfigured', 'No core found in /proc/stat', $stat['errors']);
}
if ($global == '' || $iowait == '') {
    skip('First reading, or a counter went backwards: usage comes with the next run');
}
value('cpu-busy', $global);
value('iowait', $iowait);
value('steal', $steal);
value('cores', $cores);