wip

bundles/archive/files/archive

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+if [[ '$1' == 'perform' ]]
+then
+  echo 'NON-DRY RUN'
+  DRY=''
+else
+  echo 'DRY RUN'
+  DRY='-n'
+fi
+
+% for dir, conf in dirs.items():
+# ${dir}
+gsutil ${'\\'}
+  -m ${'\\'}
+  -o 'GSUtil:parallel_process_count=${processes}' ${'\\'}
+  -o 'GSUtil:parallel_thread_count=${threads}' ${'\\'}
+  rsync ${'\\'}
+    $DRY ${'\\'}
+    -r ${'\\'}
+    -d ${'\\'}
+    -e ${'\\'}
+    -x '${'|'.join(conf['exclude'])}' ${'\\'}
+    '${dir}' ${'\\'}
+    'gs://${bucket}/${node.name}${dir}'
+
+% endfor

bundles/archive/items.py

@@ -0,0 +1,16 @@
+assert node.has_bundle('gcloud')
+
+files['/opt/archive'] = {
+    'content_type': 'mako',
+    'mode': '700',
+    'context': {
+        'dirs': node.metadata.get('archive'),
+        'bucket': node.metadata.get('gcloud/bucket'),
+        'processes': 4,
+        'threads': 16,
+    },
+    'needs': [
+        'bundle:gcloud',
+    ],
+}
+    

bundles/archive/metadata.py

@@ -0,0 +1,4 @@
+defaults = {
+    'backup': [],
+    'archive': {},
+}

bundles/gcloud/README.md

@@ -2,4 +2,6 @@
 gcloud projects add-iam-policy-binding sublimity-182017 --member 'serviceAccount:backup@sublimity-182017.iam.gserviceaccount.com' --role 'roles/storage.objectViewer'
 gcloud projects add-iam-policy-binding sublimity-182017 --member 'serviceAccount:backup@sublimity-182017.iam.gserviceaccount.com' --role 'roles/storage.objectCreator'
 gsutil -o "GSUtil:parallel_process_count=3" -o GSUtil:parallel_thread_count=4 -m rsync -r -d -e /var/vmail gs://sublimity-backup/mailserver
+gsutil config
+gsutil versioning set on gs://sublimity-backup
 ```

bundles/nextcloud/metadata.py

@@ -0,0 +1,31 @@
+defaults = {
+    'archive': { 
+        '/var/lib/nextcloud/': {
+            'exclude': [
+                '^appdata_',
+                '^updater-',
+                '^nextcloud\.log',
+                '^updater\.log',
+                '^[^/]+/cache',
+                '^[^/]+/files_versions',
+                '^[^/]+/files_trashbin',
+            ],
+        }
+    },
+}
+
+
+@metadata_reactor.provides(
+    'archive',
+)
+def exclude_hidden_files_from_archive(metadata):
+    return {
+        'archive': {
+            dir: {
+                'exclude': [
+                    '^\..*',
+                    '/\..*',
+                ],
+            } for dir, conf in metadata.get('archive').items()
+        }
+    }

groups/all.py

@@ -1,9 +1,5 @@
 {
     'metadata': {
         'dns': {},
-        'gcloud': {
-            'service_account': 'backup',
-            'project': 'sublimity-182017',
-        },
     }
 }

groups/applications/archive.py

@@ -0,0 +1,8 @@
+{
+    'supergroups': [
+        'gcloud',
+    ],
+    'bundles': [
+        'archive',
+    ],
+}

groups/applications/gcloud.py

@@ -0,0 +1,12 @@
+{
+    'bundles': [
+        'gcloud',
+    ],
+    'metadata': {
+        'gcloud': {
+            'service_account': 'backup',
+            'bucket': 'sublimity-backup',
+            'project': 'sublimity-182017',
+        },
+    },
+}

nodes/home.server.py

@@ -2,12 +2,13 @@
     'hostname': '10.0.0.2',
     'groups': [
         'debian-10',
+        'archive',
     ],
     'bundles': [
         'gitea',
+        'nextcloud',
         'postgresql',
         'wireguard',
-        'gcloud',
     ],
     'metadata': {
         'network': {