diff --git a/bundles/archive/files/archive b/bundles/archive/files/archive new file mode 100644 index 0000000..d08c714 --- /dev/null +++ b/bundles/archive/files/archive @@ -0,0 +1,27 @@ +#!/bin/bash + +if [[ '$1' == 'perform' ]] +then + echo 'NON-DRY RUN' + DRY='' +else + echo 'DRY RUN' + DRY='-n' +fi + +% for dir, conf in dirs.items(): +# ${dir} +gsutil ${'\\'} + -m ${'\\'} + -o 'GSUtil:parallel_process_count=${processes}' ${'\\'} + -o 'GSUtil:parallel_thread_count=${threads}' ${'\\'} + rsync ${'\\'} + $DRY ${'\\'} + -r ${'\\'} + -d ${'\\'} + -e ${'\\'} + -x '${'|'.join(conf['exclude'])}' ${'\\'} + '${dir}' ${'\\'} + 'gs://${bucket}/${node.name}${dir}' + +% endfor diff --git a/bundles/archive/items.py b/bundles/archive/items.py new file mode 100644 index 0000000..59d9b30 --- /dev/null +++ b/bundles/archive/items.py @@ -0,0 +1,16 @@ +assert node.has_bundle('gcloud') + +files['/opt/archive'] = { + 'content_type': 'mako', + 'mode': '700', + 'context': { + 'dirs': node.metadata.get('archive'), + 'bucket': node.metadata.get('gcloud/bucket'), + 'processes': 4, + 'threads': 16, + }, + 'needs': [ + 'bundle:gcloud', + ], +} + diff --git a/bundles/archive/metadata.py b/bundles/archive/metadata.py new file mode 100644 index 0000000..dacd3b4 --- /dev/null +++ b/bundles/archive/metadata.py @@ -0,0 +1,4 @@ +defaults = { + 'backup': [], + 'archive': {}, +} diff --git a/bundles/gcloud/README.md b/bundles/gcloud/README.md index fae07d9..d964a08 100644 --- a/bundles/gcloud/README.md +++ b/bundles/gcloud/README.md @@ -2,4 +2,6 @@ gcloud projects add-iam-policy-binding sublimity-182017 --member 'serviceAccount:backup@sublimity-182017.iam.gserviceaccount.com' --role 'roles/storage.objectViewer' gcloud projects add-iam-policy-binding sublimity-182017 --member 'serviceAccount:backup@sublimity-182017.iam.gserviceaccount.com' --role 'roles/storage.objectCreator' gsutil -o "GSUtil:parallel_process_count=3" -o GSUtil:parallel_thread_count=4 -m rsync -r -d -e /var/vmail gs://sublimity-backup/mailserver +gsutil config +gsutil versioning set on gs://sublimity-backup ``` diff --git a/bundles/nextcloud/metadata.py b/bundles/nextcloud/metadata.py new file mode 100644 index 0000000..23d3239 --- /dev/null +++ b/bundles/nextcloud/metadata.py @@ -0,0 +1,31 @@ +defaults = { + 'archive': { + '/var/lib/nextcloud/': { + 'exclude': [ + '^appdata_', + '^updater-', + '^nextcloud\.log', + '^updater\.log', + '^[^/]+/cache', + '^[^/]+/files_versions', + '^[^/]+/files_trashbin', + ], + } + }, +} + + +@metadata_reactor.provides( + 'archive', +) +def exclude_hidden_files_from_archive(metadata): + return { + 'archive': { + dir: { + 'exclude': [ + '^\..*', + '/\..*', + ], + } for dir, conf in metadata.get('archive').items() + } + } diff --git a/groups/all.py b/groups/all.py index 60446c8..2e8e687 100644 --- a/groups/all.py +++ b/groups/all.py @@ -1,9 +1,5 @@ { 'metadata': { 'dns': {}, - 'gcloud': { - 'service_account': 'backup', - 'project': 'sublimity-182017', - }, } } diff --git a/groups/applications/archive.py b/groups/applications/archive.py new file mode 100644 index 0000000..4f8d809 --- /dev/null +++ b/groups/applications/archive.py @@ -0,0 +1,8 @@ +{ + 'supergroups': [ + 'gcloud', + ], + 'bundles': [ + 'archive', + ], +} diff --git a/groups/applications/gcloud.py b/groups/applications/gcloud.py new file mode 100644 index 0000000..33f29af --- /dev/null +++ b/groups/applications/gcloud.py @@ -0,0 +1,12 @@ +{ + 'bundles': [ + 'gcloud', + ], + 'metadata': { + 'gcloud': { + 'service_account': 'backup', + 'bucket': 'sublimity-backup', + 'project': 'sublimity-182017', + }, + }, +} diff --git a/nodes/home.server.py b/nodes/home.server.py index 745f08e..1e8bd69 100644 --- a/nodes/home.server.py +++ b/nodes/home.server.py @@ -2,12 +2,13 @@ 'hostname': '10.0.0.2', 'groups': [ 'debian-10', + 'archive', ], 'bundles': [ 'gitea', + 'nextcloud', 'postgresql', 'wireguard', - 'gcloud', ], 'metadata': { 'network': {