feat(backup-freshness-check): wake the server and check once a day

The backup server sleeps most of the time, so the check was UNKNOWN.
check_by_sshmon now runs an optional wake command first; the freshness
check uses the wol-sleeper wake command, runs every 24h and retries
after 6h so a failure does not wake the server every 30s. nagios on
the icinga node may run the waker command as wol.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RywMtsL8qXG7GDY3MHq9Uh

bundles/backup-freshness-check/metadata.py

@@ -8,7 +8,11 @@ defaults = {
         'services': {
             'backup freshness': {
                 'vars.command': '/usr/lib/nagios/plugins/check_backup_freshness',
-                'check_interval': '6h',
+                'check_interval': '24h',
+                # a failed check would otherwise retry, and wake the server, every 30s
+                'retry_interval': '6h',
+                # the wakeup alone may take up to 10 minutes
+                'check_timeout': 900,
                 'vars.sudo': True,
             },
         },
@@ -16,6 +20,24 @@ defaults = {
 }
 
 
+@metadata_reactor.provides(
+    'monitoring/services/backup freshness/vars.wake',
+)
+def wake(metadata):
+    if not node.has_bundle('wol-sleeper'):
+        return {}
+
+    return {
+        'monitoring': {
+            'services': {
+                'backup freshness': {
+                    'vars.wake': metadata.get('wol-sleeper/wake_command'),
+                },
+            },
+        },
+    }
+
+
 @metadata_reactor.provides(
     'backup-freshness-check/datasets'
 )

bundles/icinga2/files/check_by_sshmon

@@ -19,6 +19,12 @@ else
     PREFIX="sudo "
 fi
 
+if [ -n "$SSHMON_WAKE" ] && ! sh -c "$SSHMON_WAKE" > /dev/null 2>&1
+then
+    echo 'check_by_sshmon: Wakeup failed' >&2
+    exit $UNKNOWN
+fi
+
 ssh sshmon@"$SSHMON_HOST" "$PREFIX$SSHMON_COMMAND"
 
 exitcode=$?

bundles/icinga2/files/conf.d/commands.conf

@@ -15,6 +15,7 @@ object CheckCommand "sshmon" {
     env.SSHMON_COMMAND = "$command$"
     env.SSHMON_HOST = "$address$"
     env.SSHMON_SUDO = "$sudo$"
+    env.SSHMON_WAKE = "$wake$"
 }
 
 

bundles/wol-waker/metadata.py

@@ -11,20 +11,35 @@ defaults = {
     'users/wol/authorized_users',
 )
 def user(metadata):
+    sleepers = [
+        sleeper
+            for sleeper in repo.nodes
+            if sleeper.has_bundle('wol-sleeper')
+            and sleeper.metadata.get('wol-sleeper/waker') == node.name
+    ]
+    commands = {
+        sleeper.metadata.get('wol-sleeper/waker_command')
+            for sleeper in sleepers
+    }
+    # icinga wakes sleepers before checks that need them awake
+    clients = {
+        f'root@{ssh_client.name}'
+            for ssh_client in repo.nodes
+            if ssh_client.dummy == False and ssh_client.has_bundle('ssh')
+    } | {
+        f"nagios@{sleeper.metadata.get('monitoring/icinga2_node')}"
+            for sleeper in sleepers
+            if sleeper.has_bundle('monitored')
+    }
+
     return {
         'users': {
             'wol': {
                 'authorized_users': {
-                    f'root@{ssh_client.name}': {
-                        'commands': {
-                            sleeper.metadata.get('wol-sleeper/waker_command')
-                                for sleeper in repo.nodes
-                                if sleeper.has_bundle('wol-sleeper')
-                                and sleeper.metadata.get('wol-sleeper/waker') == node.name
-                        }
+                    client: {
+                        'commands': commands,
                     }
-                        for ssh_client in repo.nodes
-                        if ssh_client.dummy == False and ssh_client.has_bundle('ssh')
+                        for client in clients
                 },
             },
         },