From e12e19d5eeb694ed8cd36af2880e4ec1577ebb79 Mon Sep 17 00:00:00 2001
From: cronekorkn <git@ckn.li>
Date: Thu, 2 Mar 2023 17:53:38 +0100
Subject: [PATCH] backup-server: check backup freshness

---
 .../files/check_backup_freshness              | 45 +++++++++++++++++++
 bundles/backup-server/items.py                | 12 +++++
 bundles/backup-server/metadata.py             | 32 +++++++++++++
 3 files changed, 89 insertions(+)
 create mode 100644 bundles/backup-server/files/check_backup_freshness
 create mode 100644 bundles/backup-server/items.py

diff --git a/bundles/backup-server/files/check_backup_freshness b/bundles/backup-server/files/check_backup_freshness
new file mode 100644
index 0000000..a8cb859
--- /dev/null
+++ b/bundles/backup-server/files/check_backup_freshness
@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+
+import json
+from subprocess import check_output
+from datetime import datetime, timedelta
+
+
+now = datetime.now()
+two_days_ago = now - timedelta(days=2)
+
+with open('/etc/backup-server.json', 'r') as file:
+    conf = json.load(file)
+
+local_datasets = check_output(['zfs', 'list', '-H', '-o', 'name']).decode().splitlines()
+errors = set()
+
+for dataset in conf['datasets']:
+    if f'tank/{dataset}' not in local_datasets:
+        errors.add(f'dataset "{dataset}" not present at all')
+        continue
+
+    snapshots = [
+        snapshot
+            for snapshot in check_output(['zfs', 'list', '-H', '-o', 'name', '-t', 'snapshot', f'tank/{dataset}', '-s', 'creation']).decode().splitlines()
+            if '@auto-backup_' in snapshot
+    ]
+
+    if not snapshots:
+        errors.add(f'dataset "{dataset}" has no backup snapshots')
+        continue
+
+    newest_backup_snapshot = snapshots[-1]
+    snapshot_datetime = datetime.utcfromtimestamp(
+        int(check_output(['zfs', 'list', '-p', '-H', '-o', 'creation', '-t', 'snapshot', newest_backup_snapshot]).decode())
+    )
+
+    if snapshot_datetime < two_days_ago:
+        days_ago = (now - snapshot_datetime).days
+        errors.add(f'dataset "{dataset}" has no backups sind {days_ago} days')
+        continue
+
+if errors:
+    for error in errors:
+        print(error)
+    exit(2)
diff --git a/bundles/backup-server/items.py b/bundles/backup-server/items.py
new file mode 100644
index 0000000..5f9438b
--- /dev/null
+++ b/bundles/backup-server/items.py
@@ -0,0 +1,12 @@
+from json import dumps
+from bundlewrap.metadata import MetadataJSONEncoder
+
+
+files = {
+    '/etc/backup-server.json': {
+        'content': dumps(node.metadata.get('backup-receiver'), indent=4, sort_keys=True, cls=MetadataJSONEncoder),
+    },
+    '/usr/lib/nagios/plugins/check_backup_freshness': {
+        'mode': '0755',
+    },
+}
diff --git a/bundles/backup-server/metadata.py b/bundles/backup-server/metadata.py
index acef717..4173a1e 100644
--- a/bundles/backup-server/metadata.py
+++ b/bundles/backup-server/metadata.py
@@ -6,6 +6,18 @@ defaults = {
             'rsync': {},
         },
     },
+    'backup-receiver': {
+        'datasets': {},
+    },
+    'monitoring': {
+        'services': {
+            'backup freshness': {
+                'vars.command': '/usr/lib/nagios/plugins/check_backup_freshness',
+                'check_interval': '6h',
+                'vars.sudo': True,
+            },
+        },
+    },
     'users': {
         'backup-receiver': {
             'authorized_keys': set(),
@@ -119,3 +131,23 @@ def backup_authorized_keys(metadata):
             },
         },
     }
+
+
+@metadata_reactor.provides(
+    'backup-receiver/datasets'
+)
+def check_datasets(metadata):
+    return {
+        'backup-receiver': {
+            'datasets': {
+                f"{other_node.metadata.get('id')}/{dataset}"
+                    for other_node in repo.nodes
+                    if other_node.has_bundle('backup')
+                    and other_node.has_bundle('zfs')
+                    and other_node.metadata.get('backup/server') == node.name
+                    for dataset, options in other_node.metadata.get('zfs/datasets').items()
+                    if options.get('backup', True)
+                    and not options.get('mountpoint', None) in [None, 'none']
+            },
+        },
+    }