From 9d9ac7ac5e1471e3dd6383e29bc1750da5e16914 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Fri, 23 Oct 2026 14:30:25 +0200 Subject: [PATCH] feat(backup-server): keep received snapshots by its own retention Clients deleted every auto-backup_ snapshot on the server except the newest, so the backup held one state only. prune-snapshots now keeps the newest per day, week and month (14/8/24 by default) independent of the source, and the clients call it after each receive. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RywMtsL8qXG7GDY3MHq9Uh --- bundles/backup-server/files/prune-snapshots | 85 +++++++++++++++++++++ bundles/backup-server/items.py | 14 ++++ bundles/backup-server/metadata.py | 9 +++ bundles/backup/files/backup_path_via_zfs | 7 +- 4 files changed, 110 insertions(+), 5 deletions(-) create mode 100644 bundles/backup-server/files/prune-snapshots create mode 100644 bundles/backup-server/items.py diff --git a/bundles/backup-server/files/prune-snapshots b/bundles/backup-server/files/prune-snapshots new file mode 100644 index 0000000..4cf7f8f --- /dev/null +++ b/bundles/backup-server/files/prune-snapshots @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Thin out the auto-backup_ snapshots of one received dataset, independent of the source. + +Usage: prune-snapshots [--dry-run] DATASET + +Keeps the newest snapshot per day, ISO week and month for the most recent days, weeks and months +in /etc/backup-server/retention.json, and always the newest snapshot, which the next incremental +receive builds on. Other snapshots of the dataset are never touched. Called by the clients' +backup_path_via_zfs after each successful receive, so it runs while this server is awake. +Exit codes: 0 ok, 1 zfs failed, 2 usage. +""" +from __future__ import annotations + +import datetime +import json +import re +import subprocess +import sys + +PREFIX = "auto-backup_" +CONFIG = "/etc/backup-server/retention.json" +# only datasets received from clients (tank//...), as the backup user calls this via sudo +DATASET_RE = re.compile(r"\Atank/[0-9a-f-]{36}/[A-Za-z0-9._/-]+\Z") +BUCKETS = { + "daily": lambda d: d.strftime("%Y-%m-%d"), + "weekly": lambda d: "%d-W%02d" % d.isocalendar()[:2], + "monthly": lambda d: d.strftime("%Y-%m"), +} + + +def zfs(*args: str) -> str: + return subprocess.run(["zfs", *args], capture_output=True, text=True, check=True, timeout=120).stdout + + +def keep(snapshots: list[tuple[str, int]], retention: dict[str, int]) -> set[str]: + """Names to keep from (name, creation epoch) pairs: per granularity the newest snapshot of each + of the most recent buckets, plus the newest snapshot overall.""" + newest_first = sorted(snapshots, key=lambda s: s[1], reverse=True) + kept = {newest_first[0][0]} if newest_first else set() + for granularity, count in retention.items(): + seen: set[str] = set() + for name, creation in newest_first: + bucket = BUCKETS[granularity](datetime.datetime.fromtimestamp(creation)) + if bucket not in seen and len(seen) < count: + seen.add(bucket) + kept.add(name) + return kept + + +def main(argv: list[str]) -> int: + dry_run = "--dry-run" in argv + args = [a for a in argv if a != "--dry-run"] + if len(args) != 1 or not DATASET_RE.match(args[0]): + print(__doc__, file=sys.stderr) + return 2 + dataset = args[0] + with open(CONFIG, encoding="utf-8") as fh: + retention = {k: int(v) for k, v in json.load(fh).items() if k in BUCKETS} + try: + lines = zfs("list", "-H", "-p", "-t", "snapshot", "-d", "1", "-o", "name,creation", dataset).splitlines() + except subprocess.CalledProcessError as e: + print(f"prune-snapshots: {e.stderr.strip()}", file=sys.stderr) + return 1 + snapshots = [] + for line in lines: + name, creation = line.split("\t") + if name.split("@", 1)[1].startswith(PREFIX): + snapshots.append((name, int(creation))) + kept = keep(snapshots, retention) + for name, _ in snapshots: + if name in kept: + continue + print(f"{'would destroy' if dry_run else 'destroying'} {name}") + if not dry_run: + try: + zfs("destroy", name) + except subprocess.CalledProcessError as e: + print(f"prune-snapshots: {e.stderr.strip()}", file=sys.stderr) + return 1 + print(f"{dataset}: {len(kept)} of {len(snapshots)} {PREFIX} snapshots kept") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/bundles/backup-server/items.py b/bundles/backup-server/items.py new file mode 100644 index 0000000..c2d1048 --- /dev/null +++ b/bundles/backup-server/items.py @@ -0,0 +1,14 @@ +from json import dumps + + +directories['/opt/backup-server'] = {} + +files['/opt/backup-server/prune-snapshots'] = { + 'mode': '755', +} + +directories['/etc/backup-server'] = {} + +files['/etc/backup-server/retention.json'] = { + 'content': dumps(node.metadata.get('backup-server/retention'), indent=4, sort_keys=True), +} diff --git a/bundles/backup-server/metadata.py b/bundles/backup-server/metadata.py index 3ecd90d..a72caf8 100644 --- a/bundles/backup-server/metadata.py +++ b/bundles/backup-server/metadata.py @@ -11,10 +11,19 @@ defaults = { 'authorized_keys': set(), }, }, + 'backup-server': { + # auto-backup_ snapshots kept per received dataset (prune-snapshots) + 'retention': { + 'daily': 14, + 'weekly': 8, + 'monthly': 24, + }, + }, 'sudoers': { 'backup-receiver': { '/usr/bin/rsync', '/sbin/zfs', + '/opt/backup-server/prune-snapshots', }, }, 'zfs': { diff --git a/bundles/backup/files/backup_path_via_zfs b/bundles/backup/files/backup_path_via_zfs index fddb71d..d527335 100644 --- a/bundles/backup/files/backup_path_via_zfs +++ b/bundles/backup/files/backup_path_via_zfs @@ -51,11 +51,8 @@ then zfs destroy "$destroyable_bookmark" done - # delete remote snapshots from bookmarks (except newest, even of not necessary; maybe for resuming tho) - for destroyable_snapshot in $($ssh sudo zfs list -t snapshot -H -o name "$target_dataset" | grep "^$target_dataset@$bookmark_prefix" | grep -v "$new_bookmark") - do - $ssh sudo zfs destroy "$destroyable_snapshot" - done + # the backup server thins out its snapshots by its own retention + $ssh sudo /opt/backup-server/prune-snapshots "$target_dataset" || echo "WARNING - pruning snapshots of $target_dataset failed" zfs bookmark "$source_dataset@$new_bookmark" "$source_dataset#$new_bookmark" zfs destroy "$source_dataset@$new_bookmark" # keep snapshots?