replication

// The storage replication jobs of a Proxmox VE cluster, by its API behind an API token (GET /api2/json/nodes,
// then GET /api2/json/nodes/<node>/replication for each node online): for each job, whether it fails, how many
// times in a row, and the time since its last good run.
// A job is named by its id, "<guest id>-<job number>" ("101-0"), as Proxmox VE names it.
// The address is any node of the cluster, the host's: https://<host:address>:<port>/api2/json/.
need('protocol:http');
$address = need('host:address');
$token = need('protocol:http:token');
$port = variable('proxmox:port', 'integer', array(
    'label'   => 'API port',
    'help'    => 'The port of the Proxmox VE API: 8006, unless a proxy serves it on another.',
    'default' => 8006,
));

per('job', 'Replication jobs');
output('state', 'text', array('label' => 'State', 'per' => 'job'));
output('failures', 'number', array('label' => 'Failures in a row', 'per' => 'job'));
output('lag', 'number', array('label' => 'Time since the last good run', 'unit' => 'min', 'per' => 'job'));

watch.state('failing', 'Replication failing', 'state', array('critical' => array('failing'), 'for' => '2 runs'));
watch.above('late', 'Replication late', 'lag', array('warning' => 60, 'critical' => 240));
watch.missing('missing', 'Replication job missing', 'state', array('enabled' => false));

// One GET, the token in the header Authorization: PVEAPIToken=<user>@<realm>!<token id>=<secret>. Proxmox VE
// answers {"data": ...}. The answer is not secret, its body having carried none.
function api_get($base, $path, $token) {
    $answer = http.get($base . $path, headers=array('Authorization' => 'PVEAPIToken=' . $token));
    if ($answer['status'] == 401) {
        fail('misconfigured', 'Proxmox VE refuses the API token, code 401');
    }
    if ($answer['status'] == 403) {
        fail('misconfigured', 'The API token may not read ' . $path . ', code 403');
    }
    if ($answer['status'] != 200) {
        fail('unreachable', 'Proxmox VE answers ' . $path . ' with code ' . $answer['status']);
    }
    $page = json.read($answer['body']);
    return $page['data'];
}

$base = 'https://' . $address . ':' . $port . '/api2/json';

$nodes = api_get($base, '/nodes', $token);
if (list.count($nodes) == 0) {
    fail('misconfigured', 'Proxmox VE lists no node');
}
$now = time.now();
foreach ($nodes as $node) {
    if ($node['status'] != 'online') {
        continue;
    }
    // Each node lists the jobs of the guests it holds: [{"id": "101-0", "guest": 101, "target": "pve2",
    // "schedule": "*/15", "last_sync": 1793062800, "last_try": 1793062800, "fail_count": 0, "duration": 4.2},
    // ...]; a failing job also has "error", a disabled one "disable": 1. last_sync is missing until a first
    // good run. Times are in seconds since 1970.
    $jobs = api_get($base, '/nodes/' . url.encode($node['node']) . '/replication', $token);
    foreach ($jobs as $job) {
        $id = $job['id'];
        if (list.get($job, 'disable', 0) == 1) {
            value('state', 'disabled', $id);
            continue;
        }
        $failures = list.get($job, 'fail_count', 0);
        $state = 'ok';
        if ($failures > 0) {
            $state = 'failing';
        }
        value('state', $state, $id);
        value('failures', $failures, $id);
        if (list.get($job, 'last_sync', 0) > 0) {
            value('lag', number.round(($now - $job['last_sync']) / 60), $id);
        }
    }
}