diff --git a/bundles/backup-server/files/receive-new-dataset b/bundles/backup-server/files/receive-new-dataset new file mode 100644 index 0000000..9d7279a --- /dev/null +++ b/bundles/backup-server/files/receive-new-dataset @@ -0,0 +1,3 @@ +!/bin/bash + +zfs send tank/nextcloud@test1 | ssh backup-receiver@10.0.0.5 sudo zfs recv tank/nextcloud diff --git a/bundles/backup-server/metadata.py b/bundles/backup-server/metadata.py index f5202d5..9f227b8 100644 --- a/bundles/backup-server/metadata.py +++ b/bundles/backup-server/metadata.py @@ -9,6 +9,29 @@ defaults = { } +@metadata_reactor.provides( + 'zfs/datasets' +) +def zfs(metadata): + datasets = {} + + for other_node in repo.nodes: + if ( + other_node.has_bundle('backup') and + other_node.metadata.get('backup/server') == node.name + ): + datasets[f"tank/{other_node.metadata.get('id')}/fs"] = { + 'mountpoint': f"/mnt/backups/{other_node.metadata.get('id')}", + 'backup': False, + } + + return { + 'zfs': { + 'datasets': datasets, + }, + } + + @metadata_reactor.provides( 'dns' ) diff --git a/bundles/backup/files/backup_path b/bundles/backup/files/backup_path index dcd9d5a..7510f3c 100644 --- a/bundles/backup/files/backup_path +++ b/bundles/backup/files/backup_path @@ -1,44 +1,14 @@ #!/bin/bash -set -e -set -x +path=$1 -PATH=$1 -SERVER=$(jq -r .server_hostname < /etc/backup/config.json) -UUID=$(jq -r .client_uuid < /etc/backup/config.json) - -echo "BACKUP - PATH: $PATH, SERVER: $SERVER, UUID: $UUID" - -if zfs list -H -o mountpoint | grep -q "$PATH" +if zfs list -H -o mountpoint | grep -q "$path" then - SOURCE_DATASET=$(zfs list -H -o mountpoint,name | grep -P "^$PATH\t" | cut -d $'\t' -f 2) - SOURCE_DATASET_STRIPPED=$(echo $SOURCE_DATASET | cut -f / -d 2-) - TARGET_DATASET="tank/$UUID/$DATASET_NAME" - SSH="ssh backup-receiver@$SERVER" - DATE=$(date +"%Y-%m-%d_%H:%M:%S") - BOOKMARK="ATUO_BACKUP_$DATE" - echo "BACKUP ZFS DATASET - SOURCE_DATASET: $SOURCE_DATASET, TARGET_DATASET: $TARGET_DATASET" - - if ! $SSH zfs list -t filesystem -H -o name | grep -q "^$TARGET_DATASET$" - then - echo "CREATING TARGET DATASET..." - zfs create -p -o mountpoint=none "$TARGET_DATASET" - fi - - zfs snap $SOURCE_DATASET@$BOOKMARK - zfs bookmark $SOURCE_DATASET@$BOOKMARK $SOURCE_DATASET\#$BOOKMARK - zfs destroy $SOURCE_DATASET@$BOOKMARK - - - - zfs send -i tank/nextcloud@test1 tank/nextcloud@test2 | ssh backup-receiver@$SERVER sudo zfs recv tank/nextcloud -elif test -d "$PATH" + /opt/backuo/backup_path_via_zfs "$path" +elif test -d "$path" then - TYPE=directory -elif test -f "$PATH" -then - TYPE=file + /opt/backuo/backup_path_via_zfs "$path" else - echo "UNKNOWN TYPE: $PATH" + echo "UNKNOWN PATH: $path" exit 1 fi diff --git a/bundles/backup/files/backup_path_via_rsync b/bundles/backup/files/backup_path_via_rsync new file mode 100644 index 0000000..e69de29 diff --git a/bundles/backup/files/backup_path_via_zfs b/bundles/backup/files/backup_path_via_zfs new file mode 100644 index 0000000..2fda582 --- /dev/null +++ b/bundles/backup/files/backup_path_via_zfs @@ -0,0 +1,44 @@ +#!/bin/bash + +set -e +set -x + +path=$1 +uuid=$(jq -r .client_uuid < /etc/backup/config.json) +server=$(jq -r .server_hostname < /etc/backup/config.json) +ssh="ssh -o StrictHostKeyChecking=no backup-receiver@$server" + +source_dataset=$(zfs list -H -o mountpoint,name | grep -P "^$path\t" | cut -d $'\t' -f 2) +target_dataset="tank/$uuid/$source_dataset" +new_bookmark="auto-backup_$(date +"%Y-%m-%d_%H:%M:%S")" + +echo "BACKUP ZFS DATASET - PATH: $path, SERVER: $server, UUID: $uuid, SOURCE_DATASET: $source_dataset, TARGET_DATASET: $TARGET_DATASET" + +# if ! $SSH zfs list -t filesystem -H -o name | grep -q "^$TARGET_DATASET$" +# then +# echo "CREATING TARGET DATASET..." +# $ssh sudo zfs create -p -o mountpoint=none "$target_dataset" +# fi + +zfs snap $source_dataset@$new_bookmark + +if zfs list -t bookmark -H -o name | grep '#auto-backup' | wc -l | grep -q "^0$" +then + echo "INITIAL BACKUP" + zfs send -v "$source_dataset@$new_bookmark" | $ssh sudo zfs recv $target_dataset +else + echo "INCREMENTAL BACKUP" + last_bookmark=$(zfs list -t bookmark -H -o name | sort | tail -1 | cut -d '#' -f 2) + zfs send -v -i "#$last_bookmark" "$source_dataset@$new_bookmark" | $ssh sudo zfs recv $target_dataset +fi + +if [[ "$?" == "0" ]] +then + echo "SUCCESS" + zfs bookmark "$source_dataset@$new_bookmark" "$source_dataset#$new_bookmark" + zfs destroy "$source_dataset@$new_bookmark" +else + echo "ERROR" + zfs destroy "$source_dataset@$new_bookmark" + exit 99 +fi diff --git a/bundles/backup/items.py b/bundles/backup/items.py index 1f81c47..69af814 100644 --- a/bundles/backup/items.py +++ b/bundles/backup/items.py @@ -1,11 +1,27 @@ from json import dumps +directories['/opt/backup'] = {} + +files['/opt/backup/backup_all'] = { + 'mode': '700', +} +files['/opt/backup/backup_path'] = { + 'mode': '700', +} +files['/opt/backup/backup_path_via_zfs'] = { + 'mode': '700', +} +files['/opt/backup/backup_path_via_rsync'] = { + 'mode': '700', +} + directories['/etc/backup'] = {} files['/etc/backup/config.json'] = { 'content': dumps( { 'server_hostname': repo.get_node(node.metadata.get('backup/server')).metadata.get('backup-server/hostname'), + 'client_uuid': node.metadata.get('id'), 'paths': sorted(set(node.metadata.get('backup/paths'))), }, indent=4, diff --git a/bundles/backup/metadata.py b/bundles/backup/metadata.py index 5002518..e619e12 100644 --- a/bundles/backup/metadata.py +++ b/bundles/backup/metadata.py @@ -1,6 +1,7 @@ defaults = { 'apt': { 'packages': { + 'jq': {}, 'rsync': {}, }, }, diff --git a/bundles/nextcloud/metadata.py b/bundles/nextcloud/metadata.py index c86f3b4..996105c 100644 --- a/bundles/nextcloud/metadata.py +++ b/bundles/nextcloud/metadata.py @@ -63,6 +63,9 @@ defaults = { 'datasets': { 'tank/nextcloud': { 'mountpoint': '/var/lib/nextcloud', + 'needed_by': [ + 'bundle:nextcloud', + ], }, }, }, diff --git a/bundles/users/items.py b/bundles/users/items.py index 5ddeba9..41fdb22 100644 --- a/bundles/users/items.py +++ b/bundles/users/items.py @@ -2,29 +2,27 @@ for group, config in node.metadata.get('groups', {}).items(): groups[group] = config for name, config in node.metadata.get('users').items(): - users[name] = { - k:v for k,v in config.items() if k in [ - "full_name", "gid", "groups", "home", "password_hash", "shell", "uid", - ] - } - directories[config['home']] = { 'owner': name, 'mode': '700', } files[f"{config['home']}/.ssh/id_{config['keytype']}"] = { - 'content': config['privkey'], + 'content': config['privkey'] + '\n', 'owner': name, 'mode': '0600', } files[f"{config['home']}/.ssh/id_{config['keytype']}.pub"] = { - 'content': config['pubkey'], + 'content': config['pubkey'] + '\n', 'owner': name, 'mode': '0600', } files[config['home'] + '/.ssh/authorized_keys'] = { - 'content': '\n'.join(sorted(config['authorized_keys'])), + 'content': '\n'.join(sorted(config['authorized_keys'])) + '\n', 'owner': name, 'mode': '0600', } + + users[name] = config + for option in ['authorized_keys', 'privkey', 'pubkey', 'keytype']: + users[name].pop(option, None) diff --git a/bundles/zfs/items.py b/bundles/zfs/items.py index 4c1d2ac..8aa98a7 100644 --- a/bundles/zfs/items.py +++ b/bundles/zfs/items.py @@ -26,10 +26,12 @@ svc_systemd = { }, } -zfs_datasets = node.metadata.get('zfs/datasets') +for name, config in node.metadata.get('zfs/datasets', {}).items(): + zfs_datasets[name] = config + zfs_datasets[name].pop('backup', None) -for name, attrs in node.metadata.get('zfs/pools', {}).items(): - zfs_pools[name] = attrs +for name, config in node.metadata.get('zfs/pools', {}).items(): + zfs_pools[name] = config # actions[f'pool_{name}_enable_trim'] = { # 'command': f'zpool set autotrim=on {name}', diff --git a/items/zfs_dataset.py b/items/zfs_dataset.py index 90a7886..65097e0 100644 --- a/items/zfs_dataset.py +++ b/items/zfs_dataset.py @@ -14,6 +14,10 @@ def create(node, path, options): option_list.append("-o {}={}".format(quote(option), quote(value))) option_args = " ".join(option_list) + print("zfs create {} {}".format( + option_args, + quote(path), + )) node.run( "zfs create {} {}".format( option_args, diff --git a/libs/ssh.py b/libs/ssh.py index a920d62..2f59072 100644 --- a/libs/ssh.py +++ b/libs/ssh.py @@ -18,7 +18,7 @@ def generate_ad25519_key_pair(secret): '-----BEGIN OPENSSH PRIVATE KEY-----', b64encode(deterministic_bytes).decode(), '-----END OPENSSH PRIVATE KEY-----', - ]) + '\n' + ]) public_key = privkey_bytes.public_key().public_bytes( encoding=serialization.Encoding.OpenSSH,