processes

// The processes of a Linux, by SSH: their total, the zombies, and for each process name watched ("sshd",
// "postgres") the number of its instances, their memory and their CPU time.
// Reads /proc/<pid>/stat of each process with cat, and the VmRSS line of /proc/<pid>/status with
// grep -s -H (two options of BusyBox as of GNU grep): the kernel writes them the same way on Alpine, Debian
// and Fedora, where the columns of ps change from one system to another. A process that ends during the
// reading is simply missing from the answer.
// CPU time in /proc/<pid>/stat is in hundredths of a second (USER_HZ, 100 on Linux).
need('protocol:ssh');

per('process', 'Processes', array('only' => array('sshd', 'cron', 'crond')));
output('processes', 'number', array('label' => 'Number of processes'));
output('zombies', 'number', array('label' => 'Zombie processes'));
output('instances', 'number', array('label' => 'Number of instances', 'per' => 'process'));
output('memory', 'number', array('label' => 'Memory', 'unit' => 'MiB', 'per' => 'process'));
output('cpu', 'number', array('label' => 'CPU time, as a share of one core', 'unit' => '%', 'per' => 'process'));

// No expected name by default: cron is named cron on Debian and crond on Fedora and Alpine.
watch.missing('missing', 'Process missing', 'instances');
watch.above('zombies', 'Zombies piling up', 'zombies', array('warning' => 20, 'for' => '3 runs'));
watch.above('too-many', 'Too many processes', 'processes', array('enabled' => false));
watch.above('memory-high', 'Memory of a process', 'memory', array('enabled' => false));
watch.above('cpu-high', 'CPU of a process', 'cpu', array('warning' => 90, 'for' => '3 runs', 'enabled' => false));

// One line per process: "812 (sshd) S 1 812 ...", the name in parentheses (it may hold blanks and
// parentheses: the last ")" closes it), then the state, and 12th and 13th after it the time spent in user
// and in kernel mode.
$stat = ssh.run('cat /proc/[0-9]*/stat');
$name_of = array();
$count = array();
$cpu = array();
$processes = 0;
$zombies = 0;
foreach (text.lines($stat['output']) as $line) {
    $m = text.match($line, '^([0-9]+) \((.*)\) (.*)$');
    if (list.count($m) == 0) {
        continue;
    }
    $rest = text.words($m[3]);
    if (list.count($rest) < 13) {
        continue;
    }
    $name = $m[2];
    $processes = $processes + 1;
    if ($rest[0] == 'Z') {
        $zombies = $zombies + 1;
    }
    $name_of[$m[1]] = $name;
    $count[$name] = list.get($count, $name, 0) + 1;
    $cpu[$name] = list.get($cpu, $name, 0) + $rest[11] + $rest[12];
}
if ($processes == 0) {
    fail('misconfigured', 'No process found in /proc/<pid>/stat', $stat['errors']);
}

// "/proc/812/status:VmRSS:	    6420 kB": resident memory, in KiB. Kernel threads have none.
$status = ssh.run('grep -s -H ^VmRSS: /proc/[0-9]*/status');
$memory = array();
foreach (text.lines($status['output']) as $line) {
    $m = text.match($line, '^/proc/([0-9]+)/status:VmRSS:\s+([0-9]+)');
    if (list.count($m) > 0 && list.has($name_of, $m[1])) {
        $name = $name_of[$m[1]];
        $memory[$name] = list.get($memory, $name, 0) + $m[2];
    }
}

value('processes', $processes);
value('zombies', $zombies);
foreach ($count as $name => $instances) {
    value('instances', $instances, $name);
    value('memory', number.round(list.get($memory, $name, 0) / 1024, 1), $name);
    // Hundredths of a second used per second are percents of one core. Nothing on the first run, nor
    // when an instance ended and the sum went backwards.
    $rate = counter.rate('cpu', $cpu[$name], $name);
    if ($rate != '') {
        value('cpu', number.round($rate, 1), $name);
    }
}