interfaces

// The network interfaces of a Linux, by SSH: the state, speed and errors of each link, its addresses, and
// the members of a bond; thirteen watches, each one set on the probe.
// Reads /sys/class/net with grep -s -H (two options of BusyBox as of GNU grep): the kernel writes it the
// same way on Alpine, Debian and Fedora. Counts global addresses with ip -o addr show scope global:
// iproute2 on Debian and Fedora, the ip applet of BusyBox on Alpine, which both know -o and scope.
need('protocol:ssh');

per('interface', 'Interfaces', array('ignore' => array('lo', 'docker*', 'veth*')));
output('link', 'text', array('label' => 'Link state', 'per' => 'interface'));
output('carrier-changes', 'number', array('label' => 'Link changes since boot', 'per' => 'interface'));
output('speed', 'number', array('label' => 'Negotiated speed', 'unit' => 'Mbit/s', 'per' => 'interface'));
output('duplex', 'text', array('label' => 'Duplex', 'per' => 'interface'));
output('mtu', 'number', array('label' => 'MTU', 'per' => 'interface'));
output('traffic-in', 'number', array('label' => 'Inbound traffic', 'unit' => 'bit/s', 'per' => 'interface'));
output('traffic-out', 'number', array('label' => 'Outbound traffic', 'unit' => 'bit/s', 'per' => 'interface'));
output('traffic', 'number', array('label' => 'Total traffic', 'unit' => 'bit/s', 'per' => 'interface'));
output('in-percent', 'number', array('label' => 'Inbound usage', 'unit' => '%', 'per' => 'interface'));
output('out-percent', 'number', array('label' => 'Outbound usage', 'unit' => '%', 'per' => 'interface'));
output('packets', 'number', array('label' => 'Packets', 'unit' => 'packets/s', 'per' => 'interface'));
output('errors', 'number', array('label' => 'Errors', 'unit' => 'errors/s', 'per' => 'interface'));
output('drops', 'number', array('label' => 'Drops', 'unit' => 'packets/s', 'per' => 'interface'));
output('drop-percent', 'number', array('label' => 'Share of packets dropped', 'unit' => '%', 'per' => 'interface'));
output('addresses', 'number', array('label' => 'IP addresses', 'per' => 'interface'));
output('bond-members', 'number', array('label' => 'Bond members', 'per' => 'interface'));
output('bond-members-up', 'number', array('label' => 'Active bond members', 'per' => 'interface'));

watch.state('link-down', 'Link down', 'link', array('critical' => array('down', 'lowerlayerdown')));
watch.increase('link-flapping', 'Link flapping', 'carrier-changes', array('warning' => 3, 'window' => '1 h'));
watch.custom('speed-degraded', 'Negotiated speed degraded', array(
    'help'     => 'Warning when the negotiated speed is below the expected speed; critical when the interface runs half-duplex.',
    'settings' => array(
        'expected' => array('label' => 'Expected speed', 'form' => 'number', 'unit' => 'Mbit/s', 'default' => 1000),
    ),
    'enabled'  => false,
));
watch.above('saturation-in', 'Inbound saturation', 'in-percent', array('warning' => 70, 'critical' => 90, 'for' => '5 min'));
watch.above('saturation-out', 'Outbound saturation', 'out-percent', array('warning' => 70, 'critical' => 90, 'for' => '5 min'));
watch.custom('errors', 'Errors', array(
    'help'     => 'Warning beyond a number of errors per minute; critical beyond a share of the traffic.',
    'settings' => array(
        'per-minute' => array('label' => 'Warning beyond', 'form' => 'number', 'unit' => 'errors/min', 'default' => 1),
        'share'      => array('label' => 'Critical beyond', 'form' => 'number', 'unit' => '% of traffic', 'default' => 0.1),
    ),
));
watch.above('drops', 'Packets dropped', 'drop-percent', array('warning' => 0.5, 'critical' => 2));
watch.habit('traffic-unusual', 'Unusual traffic', 'traffic', array('floor' => 10000000, 'enabled' => false));
watch.silence('mute', 'Silent interface', 'packets', array('warning' => '10 min', 'enabled' => false));
watch.missing('missing', 'Interface missing', 'link', array('enabled' => false));
watch.custom('bond-degraded', 'Bond degraded', array(
    'help'     => 'Warning when a member of a bond goes down; critical when only a given number is left.',
    'settings' => array(
        'remaining' => array('label' => 'Critical when only', 'form' => 'number', 'default' => 1),
    ),
));
watch.expect('mtu', 'Unexpected MTU', 'mtu', array('warning' => array(1500), 'enabled' => false));
watch.below('no-address', 'No IP address left', 'addresses', array('critical' => 1, 'enabled' => false));

// One line "<interface>/<file>:<value>" per readable file of /sys/class/net. The speed of a link that is
// down cannot be read, nor carrier_changes before kernel 3.15: grep -s silences them, their line is
// missing, and grep exits with an error without harm.
$files = array('operstate', 'carrier_changes', 'speed', 'duplex', 'mtu', 'bonding/slaves', 'brport/state',
    'statistics/rx_bytes', 'statistics/tx_bytes', 'statistics/rx_packets', 'statistics/tx_packets',
    'statistics/rx_errors', 'statistics/tx_errors', 'statistics/rx_dropped', 'statistics/tx_dropped');
$sys = ssh.run('cd /sys/class/net && grep -s -H . */' . text.join($files, ' */'));

// $read['eth0']['statistics/rx_bytes'] is '12345'.
$read = array();
foreach (text.lines($sys['output']) as $line) {
    $m = text.match($line, '^([^/]+)/([^:]+):(.*)$');
    if (list.count($m) > 0) {
        $read[$m[1]][$m[2]] = $m[3];
    }
}
if (list.count($read) == 0) {
    fail('misconfigured', 'No interface found in /sys/class/net', $sys['errors']);
}

// Members of a bond or a bridge carry no address: they are not judged on it.
$member_of = array();
foreach ($read as $name => $f) {
    foreach (text.words(list.get($f, 'bonding/slaves', '')) as $member) {
        $member_of[$member] = $name;
    }
    if (list.has($f, 'brport/state')) {
        $member_of[$name] = 'bridge';
    }
}

// One line per global address: "2: eth0    inet 192.0.2.10/24 ...", the name as second word.
$ip = ssh.run('ip -o addr show scope global');
if ($ip['exit'] != 0) {
    fail('misconfigured', 'The ip command failed', $ip['errors']);
}
$address_count = array();
foreach (text.lines($ip['output']) as $line) {
    $words = text.words($line);
    $address_count[$words[1]] = list.get($address_count, $words[1], 0) + 1;
}

$expected_speed = watch.setting('speed-degraded', 'expected');
$errors_per_minute = watch.setting('errors', 'per-minute');
$errors_share = watch.setting('errors', 'share');
$remaining = watch.setting('bond-degraded', 'remaining');

foreach ($read as $name => $f) {
    value('link', list.get($f, 'operstate', 'unknown'), $name);
    if (list.has($f, 'carrier_changes')) {
        value('carrier-changes', $f['carrier_changes'], $name);
    }
    if (list.has($f, 'mtu')) {
        value('mtu', $f['mtu'], $name);
    }
    if (!list.has($member_of, $name)) {
        value('addresses', list.get($address_count, $name, 0), $name);
    }

    // Negotiated speed and duplex: a link that is down has no speed (-1), or no line.
    $speed = list.get($f, 'speed', 0);
    if ($speed > 0) {
        value('speed', $speed, $name);
        if ($expected_speed != '' && $speed < $expected_speed) {
            watch.raise('speed-degraded', 'warning', $name . ' negotiated at ' . $speed . ' Mbit/s', $name);
        }
    }
    if (list.has($f, 'duplex')) {
        value('duplex', $f['duplex'], $name);
        if ($f['duplex'] == 'half') {
            watch.raise('speed-degraded', 'critical', $name . ' runs half-duplex', $name);
        }
    }

    // A bond: its members, and those whose link is up.
    if (list.has($f, 'bonding/slaves')) {
        $members = text.words($f['bonding/slaves']);
        $up = 0;
        foreach ($members as $member) {
            if (list.get(list.get($read, $member, array()), 'operstate', '') == 'up') {
                $up = $up + 1;
            }
        }
        value('bond-members', list.count($members), $name);
        value('bond-members-up', $up, $name);
        $state = $name . ': ' . $up . ' active member(s) out of ' . list.count($members);
        if ($remaining != '' && $up <= $remaining) {
            watch.raise('bond-degraded', 'critical', $state, $name);
        }
        if ($up < list.count($members)) {
            watch.raise('bond-degraded', 'warning', $state, $name);
        }
    }

    // Traffic, from the counters of this run and the previous one; nothing on the first run.
    if (!list.has($f, 'statistics/rx_bytes')) {
        continue;
    }
    $in = counter.rate('in', $f['statistics/rx_bytes'] * 8, $name);
    $out = counter.rate('out', $f['statistics/tx_bytes'] * 8, $name);
    $packets = counter.rate('packets', $f['statistics/rx_packets'] + $f['statistics/tx_packets'], $name);
    $errors = counter.rate('errors', $f['statistics/rx_errors'] + $f['statistics/tx_errors'], $name);
    $drops = counter.rate('drops', $f['statistics/rx_dropped'] + $f['statistics/tx_dropped'], $name);
    if ($in == '' || $out == '' || $packets == '' || $errors == '' || $drops == '') {
        continue;
    }
    value('traffic-in', number.round($in), $name);
    value('traffic-out', number.round($out), $name);
    value('traffic', number.round($in + $out), $name);
    value('packets', number.round($packets, 2), $name);
    value('errors', number.round($errors, 3), $name);
    value('drops', number.round($drops, 3), $name);
    if ($packets + $drops > 0) {
        value('drop-percent', number.round(100 * $drops / ($packets + $drops), 2), $name);
    }
    if ($speed > 0) {
        // Negotiated speed is in Mbit/s: one percent of it is 10,000 bit/s.
        value('in-percent', number.round($in / ($speed * 10000), 1), $name);
        value('out-percent', number.round($out / ($speed * 10000), 1), $name);
    }

    // Errors: per minute for the warning, as a share of the traffic for the critical.
    if ($errors_per_minute != '' && $errors * 60 > $errors_per_minute) {
        watch.raise('errors', 'warning', $name . ': ' . number.round($errors * 60, 1) . ' errors per minute', $name);
    }
    if ($errors_share != '' && $packets + $errors > 0) {
        $share = 100 * $errors / ($packets + $errors);
        if ($share > $errors_share) {
            watch.raise('errors', 'critical', $name . ': ' . number.round($share, 2) . ' % of traffic in error', $name);
        }
    }
}