pods

// The pods of a Kubernetes cluster, or of one namespace, by its API with the token of a service account (GET
// /api/v1/pods, or /api/v1/namespaces/<namespace>/pods): the phase of each pod, why a container of it waits
// to run (a crash loop, an image that does not pull), and how many times its containers have restarted.
// A pod is named <namespace>/<pod>, so that the selection of the probe keeps or leaves out a namespace
// (kube-system/*). The pods that have finished their work (Succeeded, the pods of a job done) are not read.
// The address is the API server's, the host's: https://<host:address>:<port>/.
need('protocol:http');
$address = need('host:address');
$token = need('protocol:http:token');
$port = variable('kubernetes:port', 'integer', array(
    'label'   => 'API port',
    'help'    => 'The port of the Kubernetes API server: 6443 for a cluster set up by kubeadm, often 443 for a managed one.',
    'default' => 6443,
));
$namespace = variable('kubernetes:namespace', 'text', array(
    'label'    => 'Namespace',
    'help'     => 'Only the pods of this namespace, for a service account allowed in it alone. Empty: all namespaces.',
    'required' => false,
));

per('pod', 'Pods');
output('phase', 'text', array('label' => 'Phase', 'per' => 'pod'));
output('waiting', 'text', array('label' => 'Container waiting for', 'per' => 'pod'));
output('restarts', 'number', array('label' => 'Container restarts', 'per' => 'pod'));

watch.state('failed', 'Pod failed or lost', 'phase', array('warning' => array('Failed', 'Unknown')));
watch.state('pending', 'Pod pending too long', 'phase', array('warning' => array('Pending'), 'for' => '15 min'));
watch.state('cannot-run', 'Container cannot run', 'waiting', array(
    'warning'  => array('ImagePullBackOff', 'ErrImagePull', 'InvalidImageName', 'CreateContainerConfigError',
        'CreateContainerError', 'RunContainerError'),
    'critical' => array('CrashLoopBackOff'),
));
watch.increase('restarts', 'Containers restarting', 'restarts', array('warning' => 3, 'critical' => 10, 'window' => '1 h'));

// The items of a list, read page by page: the API cuts a list at limit items and gives, under
// metadata.continue, the token of the next page, until the last. $query, when not empty, is added to the
// parameters of the address. The token goes in the header Authorization: Bearer; the answer is not secret,
// its body having carried none.
function api_list($base, $path, $query, $token) {
    $items = array();
    $next = '';
    while (true) {
        $url = $base . $path . '?limit=500';
        if ($query != '') {
            $url = $url . '&' . $query;
        }
        if ($next != '') {
            $url = $url . '&continue=' . url.encode($next);
        }
        $answer = http.get($url, headers=array('Authorization' => 'Bearer ' . $token));
        if ($answer['status'] == 401) {
            fail('misconfigured', 'Kubernetes refuses the token, code 401');
        }
        if ($answer['status'] == 403) {
            fail('misconfigured', 'The service account may not list ' . $path . ', code 403');
        }
        if ($answer['status'] != 200) {
            fail('unreachable', 'Kubernetes answers ' . $path . ' with code ' . $answer['status']);
        }
        $page = json.read($answer['body']);
        foreach ($page['items'] as $item) {
            $items = list.add($items, $item);
        }
        $next = list.get($page['metadata'], 'continue', '');
        if ($next == '') {
            break;
        }
    }
    return $items;
}

$path = '/api/v1/pods';
if ($namespace != '') {
    $path = '/api/v1/namespaces/' . url.encode($namespace) . '/pods';
}
$pods = api_list('https://' . $address . ':' . $port, $path, 'fieldSelector=' . url.encode('status.phase!=Succeeded'), $token);

// {"metadata": {"namespace": "shop", "name": "web-7d4b9c-x2kq"}, "status": {"phase": "Running",
// "initContainerStatuses": [...], "containerStatuses": [{"name": "web", "restartCount": 4, "state":
// {"waiting": {"reason": "CrashLoopBackOff"}}}]}}. The phase is Pending, Running, Succeeded, Failed or
// Unknown. A container that waits gives its reason; a crash loop is told before any other.
foreach ($pods as $pod) {
    $name = $pod['metadata']['namespace'] . '/' . $pod['metadata']['name'];
    $status = $pod['status'];
    value('phase', list.get($status, 'phase', 'Unknown'), $name);

    $waiting = 'none';
    $restarts = 0;
    $containers = list.get($status, 'initContainerStatuses', array());
    foreach (list.get($status, 'containerStatuses', array()) as $container) {
        $restarts = $restarts + list.get($container, 'restartCount', 0);
        $containers = list.add($containers, $container);
    }
    foreach ($containers as $container) {
        $state = list.get($container, 'state', array());
        if (list.has($state, 'waiting')) {
            $reason = list.get($state['waiting'], 'reason', '');
            if ($reason != '' && ($waiting == 'none' || $reason == 'CrashLoopBackOff')) {
                $waiting = $reason;
            }
        }
    }
    value('waiting', $waiting, $name);
    value('restarts', $restarts, $name);
}