replication

// The replication of a Redis server, by SSH and redis-cli INFO on its server, on either side. On a master, one
// link per replica connected: its state and its lag. On a replica, one link, "master": the state of its link
// to the master and the seconds since they last talked. The role of the server, and the number of links. The
// password, when Redis asks for one, stays on the server.
need('protocol:ssh');
$port = variable('redis:port', 'integer', array(
    'label'   => 'Port',
    'help'    => 'The port Redis listens on, on the server itself.',
    'default' => 6379,
));
$password_file = variable('redis:password-file', 'text', array(
    'label'    => 'Password file',
    'help'     => 'The path, on the server, of a file the SSH account reads and that holds the Redis password alone, for instance /home/monitoring/.redis-password. Empty: no password.',
    'required' => false,
));

per('link', 'Replication links');
output('role', 'text', array('label' => 'Role'));
output('links', 'number', array('label' => 'Replication links'));
output('state', 'text', array('label' => 'State', 'per' => 'link'));
output('lag', 'number', array('label' => 'Seconds since last exchange', 'unit' => 's', 'per' => 'link'));

watch.below('no-link', 'No replication link', 'links', array('critical' => 1));
watch.expect('down', 'Replication link down', 'state', array('critical' => array('online', 'up'), 'for' => '3 runs'));
watch.above('lag', 'Replication late', 'lag', array('warning' => 10, 'critical' => 60));
watch.expect('role', 'Role changed', 'role', array('enabled' => false));
watch.missing('missing', 'Replica missing', 'state', array('enabled' => false));

// INFO through redis-cli, to the Redis of the server on its port: one line "field:value" per figure. A
// password goes from its file into REDISCLI_AUTH, which redis-cli reads, by the shell of the server: it is
// never written on a command line, and never leaves the server. LC_ALL=C keeps the messages in English.
function redis_info($port, $password_file) {
    $command = 'LC_ALL=C redis-cli -p ' . $port;
    if ($password_file != '') {
        if (list.count(text.match($password_file, '^/[A-Za-z0-9_./-]+$')) == 0) {
            fail('misconfigured', 'The password file must be an absolute path of letters, digits and . _ / -');
        }
        $command = 'REDISCLI_AUTH="$(cat \'' . $password_file . '\')" ' . $command;
    }
    $read = ssh.run($command . ' INFO');
    $said = $read['output'] . $read['errors'];
    if (!text.contains($read['output'], 'redis_version:')) {
        if (text.contains($said, 'Could not connect') || text.contains($said, 'Connection refused')) {
            fail('critical', 'Redis is not running', $said);
        }
        if (list.count(text.match($said, 'NOAUTH|WRONGPASS|NOPERM|invalid password')) > 0) {
            fail('misconfigured', 'Redis refuses the account', $said);
        }
        fail('misconfigured', 'redis-cli did not answer INFO', $said);
    }
    $info = array();
    foreach (text.lines($read['output']) as $line) {
        $m = text.match($line, '^([a-z0-9_]+):(.*)$');
        if (list.count($m) > 0) {
            $info[$m[1]] = $m[2];
        }
    }
    return $info;
}

$info = redis_info($port, $password_file);
$links = 0;

// A master lists each replica connected: "slave0:ip=192.0.2.21,port=6379,state=online,offset=1093,lag=0",
// named here <ip>:<port>; a replica can itself have replicas (chained replication).
foreach ($info as $field => $text) {
    if (list.count(text.match($field, '^slave[0-9]+$')) == 0) {
        continue;
    }
    $fields = array();
    foreach (text.split($text, ',') as $pair) {
        $m = text.match($pair, '^([a-z_]+)=(.*)$');
        if (list.count($m) > 0) {
            $fields[$m[1]] = $m[2];
        }
    }
    $name = list.get($fields, 'ip', '') . ':' . list.get($fields, 'port', '');
    $links = $links + 1;
    value('state', list.get($fields, 'state', ''), $name);
    if (text.is_number(list.get($fields, 'lag', ''))) {
        value('lag', $fields['lag'], $name);
    }
}

// A replica: master_link_status up or down, and master_last_io_seconds_ago, -1 while the link is down.
if ($info['role'] == 'slave') {
    $links = $links + 1;
    value('state', $info['master_link_status'], 'master');
    if ($info['master_last_io_seconds_ago'] + 0 >= 0) {
        value('lag', $info['master_last_io_seconds_ago'], 'master');
    }
}
value('role', $info['role']);
value('links', $links);