backup-server: check backup freshness

bundles/backup-server/files/check_backup_freshness

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+
+import json
+from subprocess import check_output
+from datetime import datetime, timedelta
+
+
+now = datetime.now()
+two_days_ago = now - timedelta(days=2)
+
+with open('/etc/backup-server.json', 'r') as file:
+    conf = json.load(file)
+
+local_datasets = check_output(['zfs', 'list', '-H', '-o', 'name']).decode().splitlines()
+errors = set()
+
+for dataset in conf['datasets']:
+    if f'tank/{dataset}' not in local_datasets:
+        errors.add(f'dataset "{dataset}" not present at all')
+        continue
+
+    snapshots = [
+        snapshot
+            for snapshot in check_output(['zfs', 'list', '-H', '-o', 'name', '-t', 'snapshot', f'tank/{dataset}', '-s', 'creation']).decode().splitlines()
+            if '@auto-backup_' in snapshot
+    ]
+
+    if not snapshots:
+        errors.add(f'dataset "{dataset}" has no backup snapshots')
+        continue
+
+    newest_backup_snapshot = snapshots[-1]
+    snapshot_datetime = datetime.utcfromtimestamp(
+        int(check_output(['zfs', 'list', '-p', '-H', '-o', 'creation', '-t', 'snapshot', newest_backup_snapshot]).decode())
+    )
+
+    if snapshot_datetime < two_days_ago:
+        days_ago = (now - snapshot_datetime).days
+        errors.add(f'dataset "{dataset}" has no backups sind {days_ago} days')
+        continue
+
+if errors:
+    for error in errors:
+        print(error)
+    exit(2)

bundles/backup-server/items.py

@@ -0,0 +1,12 @@
+from json import dumps
+from bundlewrap.metadata import MetadataJSONEncoder
+
+
+files = {
+    '/etc/backup-server.json': {
+        'content': dumps(node.metadata.get('backup-receiver'), indent=4, sort_keys=True, cls=MetadataJSONEncoder),
+    },
+    '/usr/lib/nagios/plugins/check_backup_freshness': {
+        'mode': '0755',
+    },
+}

bundles/backup-server/metadata.py

@@ -6,6 +6,18 @@ defaults = {
             'rsync': {},
         },
     },
+    'backup-receiver': {
+        'datasets': {},
+    },
+    'monitoring': {
+        'services': {
+            'backup freshness': {
+                'vars.command': '/usr/lib/nagios/plugins/check_backup_freshness',
+                'check_interval': '6h',
+                'vars.sudo': True,
+            },
+        },
+    },
     'users': {
         'backup-receiver': {
             'authorized_keys': set(),
@@ -119,3 +131,23 @@ def backup_authorized_keys(metadata):
             },
         },
     }
+
+
+@metadata_reactor.provides(
+    'backup-receiver/datasets'
+)
+def check_datasets(metadata):
+    return {
+        'backup-receiver': {
+            'datasets': {
+                f"{other_node.metadata.get('id')}/{dataset}"
+                    for other_node in repo.nodes
+                    if other_node.has_bundle('backup')
+                    and other_node.has_bundle('zfs')
+                    and other_node.metadata.get('backup/server') == node.name
+                    for dataset, options in other_node.metadata.get('zfs/datasets').items()
+                    if options.get('backup', True)
+                    and not options.get('mountpoint', None) in [None, 'none']
+            },
+        },
+    }