wip

.envrc

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+python3 -m venv .venv
+source ./.venv/bin/activate
+unset PS1

.gitignore

@@ -1 +1,2 @@
 .secrets.cfg*
+.venv

.python-version

@@ -0,0 +1 @@
+3.9.0

bin/script-template

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from bundlewrap.repo import Repository
+from os.path import realpath, dirname
+
+repo = Repository(dirname(dirname(realpath(__file__))))

bundles/apt/README.md

@@ -0,0 +1,13 @@
+```python
+{
+    'apt': {
+        'packages': {
+            'apt-transport-https': {},
+        },
+        'sources': [
+            # place key under data/apt/keys/packages.cloud.google.com.{asc|gpg}
+            'deb https://packages.cloud.google.com/apt cloud-sdk main',
+        ],
+    },
+}
+```

bundles/apt/items.py

@@ -1,3 +1,35 @@
+from os.path import join
+from urllib.parse import urlparse
+from glob import glob
+from os.path import join, basename
+
+directories = {
+    '/etc/apt/sources.list.d': {
+        'purge': True,
+        'triggers': {
+            'action:apt_update',
+        },
+    },
+    '/etc/apt/trusted.gpg.d': {
+        'purge': True,
+        'triggers': {
+            'action:apt_update',
+        },
+    },
+    '/etc/apt/preferences.d': {
+        'purge': True,
+        'triggers': {
+            'action:apt_update',
+        },
+    },
+}
+
+files = {
+    '/etc/apt/sources.list': {
+        'content': '# managed'
+    },
+}
+
 actions = {
     'apt_update': {
         'command': 'apt-get update',
@@ -9,5 +41,57 @@ actions = {
     },
 }
 
+hosts = {}
+
+for source_string in node.metadata.get('apt/sources'):
+    source = repo.libs.apt.AptSource(source_string)
+    hosts\
+        .setdefault(source.url.hostname, set())\
+        .add(source)
+
+for host, sources in hosts.items():
+    keyfile = basename(glob(join(repo.path, 'data', 'apt', 'keys', f'{host}.*'))[0])
+    destination_path = f'/etc/apt/trusted.gpg.d/{keyfile}'
+
+    for source in sources:
+        source.options['signed-by'] = [destination_path]
+
+    files[f'/etc/apt/sources.list.d/{host}.list'] = {
+        'content': '\n'.join(
+            str(source) for source in sorted(sources)
+        ).format(
+            release=node.metadata.get('os_release')
+        ),
+        'triggers': {
+            'action:apt_update',
+        },
+    }
+    
+    files[destination_path] = {
+        'source': join(repo.path, 'data', 'apt', 'keys', keyfile),
+        'content_type': 'binary',
+        'triggers': {
+            'action:apt_update',
+        },
+    }
+
+
 for package, options in node.metadata.get('apt/packages', {}).items():    
     pkg_apt[package] = options
+
+    if options.get('backports', None):
+        pkg_apt[package].pop('backports')
+
+        files[f'/etc/apt/preferences.d/{package}'] = {
+            'content': '\n'.join([
+                f"Package: {package}",
+                f"Pin: release a={node.metadata.get('os_release')}-backports",
+                f"Pin-Priority: 900",
+            ]),
+            'needed_by': [
+                f'pkg_apt:{package}',
+            ],
+            'triggers': {
+                'action:apt_update',
+            },
+        }

bundles/apt/metadata.py

@@ -0,0 +1,6 @@
+defaults = {
+    'apt': {
+        'packages': {},
+        'sources': [],
+    },
+}

bundles/archive/README.md

@@ -0,0 +1,12 @@
+```
+defaults = {
+    'archive': {
+        '/var/important': {
+          'exclude': [
+            '\.cache/',
+            '\.log$',
+          ],
+        },
+    },
+}
+```

bundles/archive/files/archive

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+if [[ "$1" == 'perform' ]]
+then
+  echo 'NON-DRY RUN'
+  DRY=''
+else
+  echo 'DRY RUN'
+  DRY='-n'
+fi
+
+% for path, options in paths.items():
+# ${path}
+gsutil ${'\\'}
+  -m ${'\\'}
+  -o 'GSUtil:parallel_process_count=${processes}' ${'\\'}
+  -o 'GSUtil:parallel_thread_count=${threads}' ${'\\'}
+  rsync ${'\\'}
+    $DRY ${'\\'}
+    -r ${'\\'}
+    -d ${'\\'}
+    -e ${'\\'}
+% if options.get('exclude'):
+    -x '${'|'.join(options['exclude'])}' ${'\\'}
+% endif
+    '${options['encrypted_path']}' ${'\\'}
+    'gs://${bucket}/${node_id}${path}' ${'\\'}
+    2>&1 | logger -st gsutil
+% endfor

bundles/archive/files/get_file

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+FILENAME=$1
+TMPFILE=$(mktemp /tmp/archive_file.XXXXXXXXXX)
+BUCKET=$(cat /etc/gcloud/gcloud.json | jq -r .bucket)
+NODE=$(cat /etc/archive/archive.json | jq -r .node_id)
+MASTERKEY=$(cat /etc/gocryptfs/masterkey)
+
+gsutil cat "gs://$BUCKET/$NODE$FILENAME" > "$TMPFILE"
+/opt/gocryptfs-inspect/gocryptfs.py --aessiv --config=/etc/gocryptfs/gocryptfs.conf --masterkey="$MASTERKEY" "$TMPFILE"

bundles/archive/files/validate_file

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+FILENAME=$1
+
+ARCHIVE=$(/opt/archive/get_file "$FILENAME" | sha256sum)
+ORIGINAL=$(cat "$FILENAME" | sha256sum)
+
+if [[ "$ARCHIVE" == "$ORIGINAL" ]]
+then
+  echo "OK"
+  exit 0
+else
+  echo "ERROR"
+  exit 1
+fi

bundles/archive/items.py

@@ -0,0 +1,43 @@
+assert node.has_bundle('gcloud')
+assert node.has_bundle('gocryptfs')
+assert node.has_bundle('gocryptfs-inspect')
+assert node.has_bundle('systemd')
+
+from json import dumps
+
+directories['/opt/archive'] = {}
+directories['/etc/archive'] = {}
+
+files['/etc/archive/archive.json'] = {
+    'content': dumps(
+        {
+            'node_id': node.metadata.get('id'),
+            **node.metadata.get('archive'),
+        },
+        indent=4,
+        sort_keys=True
+    ),
+}
+
+files['/opt/archive/archive'] = {
+    'content_type': 'mako',
+    'mode': '700',
+    'context': {
+        'node_id': node.metadata.get('id'),
+        'paths': node.metadata.get('archive/paths'),
+        'bucket': node.metadata.get('gcloud/bucket'),
+        'processes': 4,
+        'threads': 4,
+    },
+    'needs': [
+        'bundle:gcloud',
+    ],
+}
+    
+files['/opt/archive/get_file'] = {
+    'mode': '700',
+}
+
+files['/opt/archive/validate_file'] = {
+    'mode': '700',
+}

bundles/archive/metadata.py

@@ -0,0 +1,45 @@
+defaults = {
+    'apt': {
+        'packages': {
+            'jq': {},
+        },
+    },
+    'archive': {
+        'paths': {},
+    },
+}
+
+
+@metadata_reactor.provides(
+    'archive/paths',
+)
+def paths(metadata):
+    return {
+        'archive': {
+            'paths': {
+                path: {
+                    'encrypted_path': f'/mnt/archive.enc{path}',
+                    'exclude': [
+                        '^\..*',
+                        '/\..*',
+                    ],
+                } for path in metadata.get('archive/paths')
+            },
+        }
+    }
+
+
+@metadata_reactor.provides(
+    'gocryptfs/paths',
+)
+def gocryptfs(metadata):
+    return {
+        'gocryptfs': {
+            'paths': {
+                path: {
+                    'mountpoint': options['encrypted_path'],
+                    'reverse': True,
+                } for path, options in metadata.get('archive/paths').items()
+            },
+        }
+    }

bundles/backup-server/files/receive-new-dataset

@@ -0,0 +1,3 @@
+!/bin/bash
+
+zfs send tank/nextcloud@test1 | ssh backup-receiver@10.0.0.5 sudo zfs recv tank/nextcloud

bundles/backup-server/metadata.py

@@ -0,0 +1,82 @@
+from ipaddress import ip_interface
+
+defaults = {
+    'users': {
+        'backup-receiver': {
+            'authorized_keys': [],
+        },
+    },
+}
+
+
+@metadata_reactor.provides(
+    'zfs/datasets'
+)
+def zfs(metadata):
+    datasets = {}
+    
+    for other_node in repo.nodes:
+        if (
+            other_node.has_bundle('backup') and
+            other_node.metadata.get('backup/server') == node.name
+        ):
+            datasets[f"tank/{other_node.metadata.get('id')}/fs"] = {
+                'mountpoint': f"/mnt/backups/{other_node.metadata.get('id')}",
+                'backup': False,
+            }
+            
+            if other_node.has_bundle('zfs'):
+                for path in other_node.metadata.get('backup/paths'):
+                    for dataset, config in other_node.metadata.get('zfs/datasets').items():
+                        if path == config.get('mountpoint'):
+                            datasets[f"tank/{other_node.metadata.get('id')}/{dataset}"] = {
+                                'mountpoint': 'none',
+                                'backup': False,
+                            }
+
+
+    return {
+        'zfs': {
+            'datasets': datasets,
+        },
+    }
+
+
+@metadata_reactor.provides(
+    'dns',
+)
+def dns(metadata):
+    return {
+        'dns': {
+            metadata.get('backup-server/hostname'): {
+                'A': [
+                    str(ip_interface(network['ipv4']).ip)
+                        for network in metadata.get('network').values()
+                        if 'ipv4' in network
+                ],
+                'AAAA': [
+                    str(ip_interface(network['ipv6']).ip)
+                        for network in metadata.get('network').values()
+                        if 'ipv6' in network
+                ],
+            },
+        },
+    }
+
+
+@metadata_reactor.provides(
+    'users/backup-receiver/authorized_keys'
+)
+def backup_authorized_keys(metadata):
+    return {
+        'users': {
+            'backup-receiver': {
+                'authorized_keys': [
+                    other_node.metadata.get('users/root/pubkey')
+                        for other_node in repo.nodes
+                        if other_node.has_bundle('backup')
+                        and other_node.metadata.get('backup/server') == node.name
+                ],
+            },
+        },
+    }

bundles/backup/files/backup_path

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+path=$1
+
+if zfs list -H -o mountpoint | grep -q "$path"
+then
+  /opt/backuo/backup_path_via_zfs "$path"
+elif test -d "$path"
+then
+  /opt/backuo/backup_path_via_rsync "$path"
+else
+  echo "UNKNOWN PATH: $path"
+  exit 1
+fi

bundles/backup/files/backup_path_via_zfs

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+set -e
+set -x
+
+path=$1
+uuid=$(jq -r .client_uuid < /etc/backup/config.json)
+server=$(jq -r .server_hostname < /etc/backup/config.json)
+ssh="ssh -o StrictHostKeyChecking=no backup-receiver@$server"
+
+source_dataset=$(zfs list -H -o mountpoint,name | grep -P "^$path\t" | cut -d $'\t' -f 2)
+target_dataset="tank/$uuid/$source_dataset"
+target_dataset_parent=$(echo $target_dataset | rev | cut -d / -f 2- | rev)
+bookmark_prefix="auto-backup_"
+new_bookmark="$bookmark_prefix$(date +"%Y-%m-%d_%H:%M:%S")"
+
+for var in path uuid server ssh source_dataset target_dataset target_dataset_parent new_bookmark
+do
+  [[ -z "${!var}" ]] && echo "ERROR - $var is empty" && exit 97
+done
+
+echo "BACKUP ZFS DATASET - PATH: $path, SERVER: $server, UUID: $uuid, SOURCE_DATASET: $source_dataset, TARGET_DATASET: $TARGET_DATASET"
+
+if ! $ssh sudo zfs list -t filesystem -H -o name | grep -q "^$target_dataset_parent$"
+then
+  echo "CREATING PARENT DATASET..."
+  $ssh sudo zfs create -p -o mountpoint=none "$target_dataset_parent"
+fi
+
+zfs snap "$source_dataset@$new_bookmark"
+
+if zfs list -t bookmark -H -o name | grep "^$source_dataset#$bookmark_prefix" | wc -l | grep -q "^0$"
+then
+  echo "INITIAL BACKUP"
+  # do in subshell, otherwise ctr+c will lead to 0 exitcode  
+  $(zfs send -v "$source_dataset@$new_bookmark" | $ssh sudo zfs recv -F "$target_dataset")
+else
+  echo "INCREMENTAL BACKUP"
+  last_bookmark=$(zfs list -t bookmark -H -o name | grep "^$source_dataset#$bookmark_prefix" | sort | tail -1 | cut -d '#' -f 2)
+  [[ -z "$last_bookmark" ]] && echo "ERROR - last_bookmark is empty" && exit 98
+  $(zfs send -v -i "#$last_bookmark" "$source_dataset@$new_bookmark" | $ssh sudo zfs recv "$target_dataset")
+fi
+
+if [[ "$?" == "0" ]]
+then
+  zfs bookmark "$source_dataset@$new_bookmark" "$source_dataset#$new_bookmark"
+  zfs destroy "$source_dataset@$new_bookmark"
+  echo "SUCCESS"
+else
+  zfs destroy "$source_dataset@$new_bookmark"
+  echo "ERROR"
+  exit 99
+fi

bundles/backup/items.py

@@ -0,0 +1,30 @@
+from json import dumps
+
+directories['/opt/backup'] = {}
+
+files['/opt/backup/backup_all'] = {
+    'mode': '700',
+}
+files['/opt/backup/backup_path'] = {
+    'mode': '700',
+}
+files['/opt/backup/backup_path_via_zfs'] = {
+    'mode': '700',
+}
+files['/opt/backup/backup_path_via_rsync'] = {
+    'mode': '700',
+}
+
+directories['/etc/backup'] = {}
+
+files['/etc/backup/config.json'] = {
+    'content': dumps(
+        {
+            'server_hostname': repo.get_node(node.metadata.get('backup/server')).metadata.get('backup-server/hostname'),
+            'client_uuid': node.metadata.get('id'),
+            'paths': sorted(set(node.metadata.get('backup/paths'))),
+        },
+        indent=4,
+        sort_keys=True
+    ),
+}

bundles/backup/metadata.py

@@ -0,0 +1,12 @@
+defaults = {
+    'apt': {
+        'packages': {
+            'jq': {},
+            'rsync': {},
+        },
+    },
+    'backup': {
+        'server': None,
+        'paths': [],
+    },
+}

bundles/bind/files/db

@@ -0,0 +1,23 @@
+<%!
+def column_width(column, table):
+    return max(map(lambda row: len(row[column]), table)) if table else 0    
+%>\
+$TTL 600
+@   IN  SOA        ns.sublimity.de. admin.sublimity.de. (
+        2020080302 ;Serial
+        600        ;Refresh
+        300        ;Retry
+        1209600    ;Expire
+        300        ;Negative response caching TTL
+)
+
+% for record in sorted(records, key=lambda r: (r['name'], r['type'], r['value'])):
+${(record['name'] or '@').ljust(column_width('name', records))} \
+IN \
+${record['type'].ljust(column_width('type', records))} \
+    % if record['type'] == 'TXT':
+(${' '.join('"'+record['value'][i:i+255]+'"' for i in range(0, len(record['value']), 255))})
+    % else:
+${record['value']}
+    % endif
+% endfor

bundles/bind/files/defaults

@@ -0,0 +1,2 @@
+RESOLVCONF=no
+OPTIONS="-u bind"

bundles/bind/files/named.conf

@@ -0,0 +1,2 @@
+include "/etc/bind/named.conf.options";
+include "/etc/bind/named.conf.local";

bundles/bind/files/named.conf.local

@@ -0,0 +1,26 @@
+% for view in views:
+acl "${view['name']}" {
+  ${' '.join(f'{e};' for e in view['acl'])}
+};
+% endfor
+
+% for view in views:
+view "${view['name']}" {
+  match-clients { ${view['name']}; };
+  recursion yes;
+  forward only;
+  forwarders {
+    1.1.1.1;
+    9.9.9.9;
+    8.8.8.8;
+  };
+  % for zone in zones:
+  zone "${zone}" {
+    type master;
+    file "/var/lib/bind/${view['name']}/db.${zone}";
+  };
+  % endfor
+  include "/etc/bind/named.conf.default-zones";
+  include "/etc/bind/zones.rfc1918";
+};
+% endfor

bundles/bind/files/named.conf.options

@@ -0,0 +1,10 @@
+options {
+  directory "/var/cache/bind";
+  dnssec-validation auto;
+
+  listen-on-v6 { any; };
+	allow-query { any; };
+
+  max-cache-size 30%;
+  querylog yes;
+};

bundles/bind/items.py

@@ -0,0 +1,141 @@
+from ipaddress import ip_address
+
+directories[f'/var/lib/bind'] = {
+    'purge': True,
+    'needed_by': [
+        'svc_systemd:bind9',
+    ],
+    'triggers': [
+        'svc_systemd:bind9:restart',
+    ],
+}
+
+files['/etc/default/bind9'] = {
+    'source': 'defaults',
+    'needed_by': [
+        'svc_systemd:bind9',
+    ],
+    'triggers': [
+        'svc_systemd:bind9:restart',
+    ],
+}
+
+files['/etc/bind/named.conf'] = {
+    'owner': 'root',
+    'group': 'bind',
+    'needed_by': [
+        'svc_systemd:bind9',
+    ],
+    'triggers': [
+        'svc_systemd:bind9:restart',
+    ],
+}
+files['/etc/bind/named.conf.options'] = {
+    'owner': 'root',
+    'group': 'bind',
+    'needed_by': [
+        'svc_systemd:bind9',
+    ],
+    'triggers': [
+        'svc_systemd:bind9:restart',
+    ],
+}
+
+views = [
+    {
+        'name': 'internal',
+        'is_internal': True,
+        'acl': [
+            '127.0.0.1',
+            '10.0.0.0/8',
+            '169.254.0.0/16',
+            '172.16.0.0/12',
+            '192.168.0.0/16',
+        ]
+    },
+    {
+        'name': 'external', 
+        'is_internal': False,
+        'acl': [
+            'any',
+        ]
+    },
+]
+
+files['/etc/bind/named.conf.local'] = {
+    'content_type': 'mako',
+    'context': {
+        'views': views,
+        'zones': sorted(node.metadata.get('bind/zones')),
+    },
+    'owner': 'root',
+    'group': 'bind',
+    'needed_by': [
+        'svc_systemd:bind9',
+    ],
+    'triggers': [
+        'svc_systemd:bind9:restart',
+    ],
+}
+
+def use_record(record, records, view):
+    if record['type'] in ['A', 'AAAA']:
+        if view == 'external':
+            # no internal addresses in external view
+            if ip_address(record['value']).is_private:
+                return False
+        elif view == 'internal':
+            # external addresses in internal view only, if no internal exists
+            if ip_address(record['value']).is_global:
+                for other_record in records:
+                    if (
+                        record['name'] == other_record['name'] and
+                        record['type'] == other_record['type'] and
+                        ip_address(other_record['value']).is_private
+                    ):
+                        return False
+    return True
+    
+for view in views:
+    directories[f"/var/lib/bind/{view['name']}"] = {
+        'purge': True,
+        'needed_by': [
+            'svc_systemd:bind9',
+        ],
+        'triggers': [
+            'svc_systemd:bind9:restart',
+        ],
+    }
+
+    for zone, records in node.metadata.get('bind/zones').items():
+        files[f"/var/lib/bind/{view['name']}/db.{zone}"] = {
+            'group': 'bind',
+            'source': 'db',
+            'content_type': 'mako',
+            'context': {
+                'view': view['name'],
+                'records': list(filter(
+                    lambda record: use_record(record, records, view['name']),
+                    records
+                )),
+            },
+            'needs': [
+                f"directory:/var/lib/bind/{view['name']}",
+            ],
+            'needed_by': [
+                'svc_systemd:bind9',
+            ],
+            'triggers': [
+                'svc_systemd:bind9:restart',
+            ],
+        }
+
+svc_systemd['bind9'] = {}
+
+actions['named-checkconf'] = {
+    'command': 'named-checkconf -z',
+    'unless': 'named-checkconf -z',
+    'needs': [
+        'svc_systemd:bind9',
+    ]
+}

bundles/bind/metadata.py

@@ -0,0 +1,87 @@
+from ipaddress import ip_interface
+
+
+defaults = {
+    'apt': {
+        'packages': {
+            'bind9': {},
+        },
+    },
+    'bind': {
+        'zones': {},
+    },
+}
+
+
+@metadata_reactor.provides(
+    'bind/zones',
+)
+def dns(metadata):
+    return {
+        'dns': {
+            metadata.get('bind/domain'): {
+                'A': [
+                    str(ip_interface(network['ipv4']).ip)
+                        for network in metadata.get('network').values()
+                        if 'ipv4' in network
+                ],
+                'AAAA': [
+                    str(ip_interface(network['ipv6']).ip)
+                        for network in metadata.get('network').values()
+                        if 'ipv6' in network
+                ]
+            },
+        },
+    }
+
+
+@metadata_reactor.provides(
+    'bind/zones',
+)
+def collect_records(metadata):
+    zones = {}
+    
+    for other_node in repo.nodes:
+        for fqdn, records in other_node.metadata.get('dns').items():
+            matching_zones = sorted(
+                filter(
+                    lambda potential_zone: fqdn.endswith(potential_zone),
+                    metadata.get('bind/zones').keys()
+                ),
+                key=len,
+            )
+            if matching_zones:
+                zone = matching_zones[-1]
+            else:
+                continue
+
+            name = fqdn[0:-len(zone) - 1]
+
+            for type, values in records.items():
+                for value in values:
+                    zones\
+                        .setdefault(zone, [])\
+                        .append(
+                            {'name': name, 'type': type, 'value': value}
+                        )
+    
+    return {
+        'bind': {
+            'zones': zones,
+        },
+    }
+
+
+@metadata_reactor.provides(
+    'bind/zones',
+)
+def ns_records(metadata):
+    return {
+        'bind': {
+            'zones': {
+                zone: [
+                    {'name': '@', 'type': 'NS', 'value': f"{metadata.get('bind/domain')}."},
+                ] for zone in metadata.get('bind/zones').keys()
+            },
+        },
+    }

bundles/dovecot/README.md

@@ -0,0 +1,9 @@
+DOVECOT
+=======
+
+rescan index: https://doc.dovecot.org/configuration_manual/fts/#rescan
+
+```
+    sudo -u vmail doveadm fts rescan -u 'test@mail2.sublimity.de'
+    sudo -u vmail doveadm index -u 'test@mail2.sublimity.de' -q '*'
+```

bundles/dovecot/files/decode2text.sh

@@ -0,0 +1,105 @@
+#!/bin/sh
+
+# Example attachment decoder script. The attachment comes from stdin, and
+# the script is expected to output UTF-8 data to stdout. (If the output isn't
+# UTF-8, everything except valid UTF-8 sequences are dropped from it.)
+
+# The attachment decoding is enabled by setting:
+#
+# plugin {
+#   fts_decoder = decode2text
+# }
+# service decode2text {
+#   executable = script /usr/local/libexec/dovecot/decode2text.sh
+#   user = dovecot
+#   unix_listener decode2text {
+#     mode = 0666
+#   }
+# }
+
+libexec_dir=`dirname $0`
+content_type=$1
+
+# The second parameter is the format's filename extension, which is used when
+# found from a filename of application/octet-stream. You can also add more
+# extensions by giving more parameters.
+formats='application/pdf pdf
+application/x-pdf pdf
+application/msword doc
+application/mspowerpoint ppt
+application/vnd.ms-powerpoint ppt
+application/ms-excel xls
+application/x-msexcel xls
+application/vnd.ms-excel xls
+application/vnd.openxmlformats-officedocument.wordprocessingml.document docx
+application/vnd.openxmlformats-officedocument.spreadsheetml.sheet xlsx
+application/vnd.openxmlformats-officedocument.presentationml.presentation pptx
+application/vnd.oasis.opendocument.text odt
+application/vnd.oasis.opendocument.spreadsheet ods
+application/vnd.oasis.opendocument.presentation odp
+'
+
+if [ "$content_type" = "" ]; then
+  echo "$formats"
+  exit 0
+fi
+
+fmt=`echo "$formats" | grep -w "^$content_type" | cut -d ' ' -f 2`
+if [ "$fmt" = "" ]; then
+  echo "Content-Type: $content_type not supported" >&2
+  exit 1
+fi
+
+# most decoders can't handle stdin directly, so write the attachment
+# to a temp file
+path=`mktemp`
+trap "rm -f $path" 0 1 2 3 14 15
+cat > $path
+
+xmlunzip() {
+  name=$1
+
+  tempdir=`mktemp -d`
+  if [ "$tempdir" = "" ]; then
+    exit 1
+  fi
+  trap "rm -rf $path $tempdir" 0 1 2 3 14 15
+  cd $tempdir || exit 1
+  unzip -q "$path" 2>/dev/null || exit 0
+  find . -name "$name" -print0 | xargs -0 cat |
+    $libexec_dir/xml2text
+}
+
+wait_timeout() {
+  childpid=$!
+  trap "kill -9 $childpid; rm -f $path" 1 2 3 14 15
+  wait $childpid
+}
+
+LANG=en_US.UTF-8
+export LANG
+if [ $fmt = "pdf" ]; then
+  /usr/bin/pdftotext $path - 2>/dev/null&
+  wait_timeout 2>/dev/null
+elif [ $fmt = "doc" ]; then
+  (/usr/bin/catdoc $path; true) 2>/dev/null&
+  wait_timeout 2>/dev/null
+elif [ $fmt = "ppt" ]; then
+  (/usr/bin/catppt $path; true) 2>/dev/null&
+  wait_timeout 2>/dev/null
+elif [ $fmt = "xls" ]; then
+  (/usr/bin/xls2csv $path; true) 2>/dev/null&
+  wait_timeout 2>/dev/null
+elif [ $fmt = "odt" -o $fmt = "ods" -o $fmt = "odp" ]; then
+  xmlunzip "content.xml"
+elif [ $fmt = "docx" ]; then
+  xmlunzip "document.xml"
+elif [ $fmt = "xlsx" ]; then
+  xmlunzip "sharedStrings.xml"
+elif [ $fmt = "pptx" ]; then
+  xmlunzip "slide*.xml"
+else
+  echo "Buggy decoder script: $fmt not handled" >&2
+  exit 1
+fi
+exit 0

bundles/dovecot/files/dovecot-sql.conf

@@ -8,10 +8,3 @@ password_query = SELECT CONCAT(users.name, '@', domains.name) AS user, password\
   WHERE redirect IS NULL \
   AND users.name = SPLIT_PART('%u', '@', 1) \
   AND domains.name = SPLIT_PART('%u', '@', 2)
-
-user_query = SELECT CONCAT(users.name, '@', domains.name) AS user, '/var/vmail/%u' AS home \
-  FROM users \
-  LEFT JOIN domains ON users.domain_id = domains.id \
-  WHERE redirect IS NULL \
-  AND users.name = SPLIT_PART('%u', '@', 1) \
-  AND domains.name = SPLIT_PART('%u', '@', 2)

bundles/dovecot/files/dovecot.conf

@@ -1,10 +1,17 @@
-!include conf.d/*.conf
+protocols = imap lmtp sieve
+auth_mechanisms = plain login
+mail_privileged_group = mail
+ssl = required
+ssl_cert = </var/lib/dehydrated/certs/${node.metadata.get('mailserver/hostname')}/fullchain.pem
+ssl_key = </var/lib/dehydrated/certs/${node.metadata.get('mailserver/hostname')}/privkey.pem
+ssl_dh = </etc/dovecot/dhparam.pem
+ssl_client_ca_dir = /etc/ssl/certs
+mail_location = maildir:~
+mail_plugins = fts fts_xapian
 
 namespace inbox {
-  separator = .
-  type = private
   inbox = yes
-  location =
+  separator = .
   mailbox Drafts {
     auto = subscribe 
     special_use = \Drafts
@@ -12,127 +19,116 @@ namespace inbox {
   mailbox Junk {
     auto = create 
     special_use = \Junk
-    autoexpunge = 30d
+  }
+  mailbox Trash {
+    auto = subscribe 
+    special_use = \Trash
   }
   mailbox Sent {
     auto = subscribe 
     special_use = \Sent
   }
-  mailbox Trash {
-    auto = subscribe
-    special_use = \Trash
-    autoexpunge = 360d
-  }
-  prefix =
-}
-
-mail_location = maildir:/var/vmail/%u
-protocols = imap lmtp sieve
-
-ssl = yes
-ssl_cert = </var/lib/dehydrated/certs/${node.metadata.get('mailserver/hostname')}/fullchain.pem
-ssl_key = </var/lib/dehydrated/certs/${node.metadata.get('mailserver/hostname')}/privkey.pem
-ssl_dh = </etc/dovecot/ssl/dhparam.pem
-ssl_min_protocol = TLSv1.2
-ssl_cipher_list = EECDH+AESGCM:EDH+AESGCM
-ssl_prefer_server_ciphers = yes
-
-login_greeting = IMAPd ready
-auth_mechanisms = plain login
-disable_plaintext_auth = yes
-mail_plugins = $mail_plugins zlib
-
-plugin {
-   zlib_save_level = 6
-   zlib_save = gz
-
-   sieve_plugins = sieve_imapsieve sieve_extprograms
-   sieve_dir = /var/vmail/sieve/%u/
-   sieve = /var/vmail/sieve/%u.sieve
-   sieve_pipe_bin_dir = /var/vmail/sieve/bin
-   sieve_extensions = +vnd.dovecot.pipe
-
-   old_stats_refresh = 30 secs
-   old_stats_track_cmds = yes
-
-% if node.has_bundle('rspamd'):
-   sieve_before = /var/vmail/sieve/global/spam-global.sieve
-
-   # From elsewhere to Spam folder
-   imapsieve_mailbox1_name = Junk
-   imapsieve_mailbox1_causes = COPY
-   imapsieve_mailbox1_before = file:/var/vmail/sieve/global/learn-spam.sieve
-
-   # From Spam folder to elsewhere
-   imapsieve_mailbox2_name = *
-   imapsieve_mailbox2_from = Junk
-   imapsieve_mailbox2_causes = COPY
-   imapsieve_mailbox2_before = file:/var/vmail/sieve/global/learn-ham.sieve
-% endif
-}
-
-service auth {
-   unix_listener /var/spool/postfix/private/auth {
-       mode = 0660
-       user = postfix
-       group = postfix
-   }
-
-   unix_listener auth-userdb {
-       mode = 0660
-       user = nobody
-       group = nogroup
-   }
-}
-
-service lmtp {
-   unix_listener /var/spool/postfix/private/dovecot-lmtp {
-       group = postfix
-       mode = 0600
-       user = postfix
-   }
-}
-
-service imap {
-   executable = imap
-}
-
-service imap-login {
-   service_count = 1
-   process_min_avail = 8
-   vsz_limit = 64M
-}
-
-service managesieve-login {
-   inet_listener sieve {
-       port = 4190
-   }
-}
-
-userdb {
-   driver = sql
-   args = /etc/dovecot/dovecot-sql.conf
 }
 
 passdb {
-   driver = sql
-   args = /etc/dovecot/dovecot-sql.conf
+  driver = sql
+  args = /etc/dovecot/dovecot-sql.conf
+}
+userdb {
+  driver = static
+  args = uid=vmail gid=vmail home=/var/vmail/%u
 }
 
-protocol lmtp {
-   mail_plugins = $mail_plugins sieve
-   postmaster_address = ${admin_email}
+service auth {
+  unix_listener /var/spool/postfix/private/auth {
+    mode = 0660
+    user = postfix
+    group = postfix
+  }
+}
+service lmtp {
+  unix_listener /var/spool/postfix/private/dovecot-lmtp {
+    mode = 0600
+    user = postfix
+    group = postfix
+  }
+}
+service stats {
+  unix_listener stats-reader {
+    user = vmail
+    group = vmail
+    mode = 0660
+  }
+  unix_listener stats-writer {
+    user = vmail
+    group = vmail
+    mode = 0660
+  }
+}
+service managesieve-login {
+  inet_listener sieve {
+  }
+  process_min_avail = 0
+  service_count = 1
+  vsz_limit = 64 M
+}
+service managesieve {
+  process_limit = 100
 }
 
 protocol imap {
-   mail_plugins = $mail_plugins imap_zlib imap_sieve imap_old_stats
+   mail_plugins = $mail_plugins imap_sieve
    mail_max_userip_connections = 50
    imap_idle_notify_interval = 29 mins
 } 
-
-protocol sieve {
-   plugin {
-       sieve = /var/vmail/sieve/%u.sieve
-       sieve_storage = /var/vmail/sieve/%u/
-   }
+protocol lmtp {
+   mail_plugins = $mail_plugins sieve
+} 
+protocol sieve {
+  plugin {
+    sieve = /var/vmail/sieve/%u.sieve
+    sieve_storage = /var/vmail/sieve/%u/
+  }
+}
+
+# fulltext search
+plugin {
+    fts = xapian
+    fts_xapian = partial=3 full=20 verbose=0
+    fts_autoindex = yes
+    fts_enforced = yes
+    # Index attachements
+    fts_decoder = decode2text
+}
+service indexer-worker {
+    vsz_limit = ${indexer_ram}
+}
+service decode2text {
+    executable = script /usr/local/libexec/dovecot/decode2text.sh
+    user = dovecot
+    unix_listener decode2text {
+        mode = 0666
+    }
+}
+
+# spam filter
+plugin {
+  sieve_plugins = sieve_imapsieve sieve_extprograms
+  sieve_dir = /var/vmail/sieve/%u/
+  sieve = /var/vmail/sieve/%u.sieve
+  sieve_pipe_bin_dir = /var/vmail/sieve/
+  sieve_extensions = +vnd.dovecot.pipe 
+
+  sieve_before = /var/vmail/sieve/global/spam-global.sieve
+
+  # From elsewhere to Spam folder
+  imapsieve_mailbox1_name = Junk
+  imapsieve_mailbox1_causes = COPY
+  imapsieve_mailbox1_before = file:/var/vmail/sieve/global/learn-spam.sieve
+
+  # From Spam folder to elsewhere
+  imapsieve_mailbox2_name = *
+  imapsieve_mailbox2_from = Junk
+  imapsieve_mailbox2_causes = COPY
+  imapsieve_mailbox2_before = file:/var/vmail/sieve/global/learn-ham.sieve
 }

bundles/dovecot/files/learn-ham.sieve

@@ -0,0 +1,7 @@
+require ["vnd.dovecot.pipe", "copy", "imapsieve", "variables"];
+
+if string "${mailbox}" "Trash" {
+  stop;
+}
+
+pipe :copy "rspamd-learn-ham.sh";

bundles/dovecot/files/learn-spam.sieve

@@ -0,0 +1,3 @@
+require ["vnd.dovecot.pipe", "copy", "imapsieve"];
+
+pipe :copy "rspamd-learn-spam.sh";

bundles/dovecot/files/spam-global.sieve

@@ -0,0 +1,6 @@
+require ["fileinto", "mailbox"];
+
+if header :contains "X-Spam" "Yes" {
+ fileinto :create "Junk";
+ stop;
+}

bundles/dovecot/items.py

@@ -1,6 +1,24 @@
 assert node.has_bundle('mailserver')
 
+groups['vmail'] = {}
+
+users['vmail'] = {
+    'home': '/var/vmail',
+    'needs': [
+        'group:vmail',
+    ],
+}
 directories = {
+    '/etc/dovecot': {
+        'purge': True,
+    },
+    '/etc/dovecot/conf.d': {
+        'purge': True,
+        'needs': [
+            'pkg_apt:dovecot-sieve',
+            'pkg_apt:dovecot-managesieved',
+        ]
+    },
     '/etc/dovecot/ssl': {},
     '/var/vmail': {
         'owner': 'vmail',
@@ -8,23 +26,12 @@ directories = {
     }
 }
 
-# groups['vmail'] = {
-#     'gid': 5000,
-# }
-# 
-# users['vmail'] = {
-#     'uid': 5000,
-#     'home': '/var/vmail',
-#     'needs': [
-#         'group:vmail',
-#     ]
-# }
-
 files = {
     '/etc/dovecot/dovecot.conf': {
         'content_type': 'mako',
         'context': {
             'admin_email': node.metadata.get('mailserver/admin_email'),
+            'indexer_ram': node.metadata.get('dovecot/indexer_ram'),
         },
         'needs': {
             'pkg_apt:'
@@ -43,15 +50,41 @@ files = {
             'svc_systemd:dovecot:restart',
         },
     },
+    '/etc/dovecot/dhparam.pem': {
+        'content_type': 'any',
+    },
+    '/etc/dovecot/dovecot-sql.conf': {
+        'content_type': 'mako',
+        'context': node.metadata.get('mailserver/database'),
+        'needs': {
+            'pkg_apt:'
+        },
+        'triggers': {
+            'svc_systemd:dovecot:restart',
+        },
+    },
+    '/var/mail/vmail/sieve/global/learn-ham.sieve': {
+        'owner': 'nobody',
+        'group': 'nogroup',
+    },
+    '/var/mail/vmail/sieve/global/learn-spam.sieve': {
+        'owner': 'nobody',
+        'group': 'nogroup',
+    },
+    '/var/mail/vmail/sieve/global/spam-global.sieve': {
+        'owner': 'nobody',
+        'group': 'nogroup',
+    },
 }
 
 actions = {
     'dovecot_generate_dhparam': {
-        'command': 'openssl dhparam -out /etc/dovecot/ssl/dhparam.pem 2048',
-        'unless': 'test -f /etc/dovecot/ssl/dhparam.pem',
+        'command': 'openssl dhparam -out /etc/dovecot/dhparam.pem 2048',
+        'unless': 'test -f /etc/dovecot/dhparam.pem',
         'cascade_skip': False,
         'needs': {
-            'pkg_apt:'
+            'pkg_apt:',
+            'directory:/etc/dovecot/ssl',
         },
         'triggers': {
             'svc_systemd:dovecot:restart',
@@ -69,3 +102,11 @@ svc_systemd = {
         },
     },
 }
+
+# fulltext search
+
+directories['/usr/local/libexec/dovecot'] = {}
+files['/usr/local/libexec/dovecot/decode2text.sh'] = {
+    'owner': 'dovecot',
+    'mode': '500',
+}

bundles/dovecot/metadata.py

@@ -1,13 +1,16 @@
-from bundlewrap.metadata import atomic
-
 defaults = {
     'apt': {
         'packages': {
-            'dovecot-imapd': {},
-            'dovecot-lmtpd': {},
+            'dovecot-imapd':        {},
+            'dovecot-pgsql':        {},
+            'dovecot-lmtpd':        {},
+            # spam filtering
+            'dovecot-sieve':        {},
             'dovecot-managesieved': {},
-            'dovecot-pgsql': {},
-            'dovecot-sieve': {},
+            # fulltext search
+            'dovecot-fts-xapian':   {}, # buster-backports
+            'poppler-utils':        {}, # pdftotext
+            'catdoc':               {}, # catdoc, catppt, xls2csv
         },
     },
     'letsencrypt': {
@@ -21,5 +24,14 @@ defaults = {
             'dbuser': 'mailserver',
         },
     },
-
 }
+
+@metadata_reactor.provides(
+    'dovecot/indexer_ram',
+)
+def indexer_ram(metadata):
+    return {
+        'dovecot': {
+            'indexer_ram': str(metadata.get('vm/ram')//2)+ 'M',
+        },
+    }

bundles/gcloud/README.md

@@ -0,0 +1,12 @@
+```
+gcloud projects add-iam-policy-binding sublimity-182017 --member 'serviceAccount:backup@sublimity-182017.iam.gserviceaccount.com' --role 'roles/storage.objectViewer'
+gcloud projects add-iam-policy-binding sublimity-182017 --member 'serviceAccount:backup@sublimity-182017.iam.gserviceaccount.com' --role 'roles/storage.objectCreator'
+gcloud projects add-iam-policy-binding sublimity-182017 --member 'serviceAccount:backup@sublimity-182017.iam.gserviceaccount.com' --role 'roles/storage.objectAdmin'
+gsutil -o "GSUtil:parallel_process_count=3" -o GSUtil:parallel_thread_count=4 -m rsync -r -d -e /var/vmail gs://sublimity-backup/mailserver
+gsutil config
+gsutil versioning set on gs://sublimity-backup
+
+
+
+gcsfuse --key-file /root/.config/gcloud/service_account.json sublimity-backup gcsfuse
+```

bundles/gcloud/items.py

@@ -0,0 +1,43 @@
+from os.path import join
+from json import dumps
+
+service_account = node.metadata.get('gcloud/service_account')
+project = node.metadata.get('gcloud/project')
+
+directories[f'/etc/gcloud'] = {
+    'purge': True,
+}
+
+files['/etc/gcloud/gcloud.json'] = {
+    'content': dumps(
+        node.metadata.get('gcloud'),
+        indent=4,
+        sort_keys=True
+    ),
+}
+
+files['/etc/gcloud/service_account.json'] = {
+    'content': repo.vault.decrypt_file(
+        join(repo.path, 'data', 'gcloud', 'service_accounts', f'{service_account}@{project}.json.enc')
+    ),
+    'mode': '500',
+    'needs': [
+        'pkg_apt:google-cloud-sdk',
+    ],
+}
+
+actions['gcloud_activate_service_account'] = {
+    'command': 'gcloud auth activate-service-account --key-file /etc/gcloud/service_account.json',
+    'unless': f"gcloud auth list | grep -q '^\*[[:space:]]*{service_account}@{project}.iam.gserviceaccount.com'",
+    'needs': [
+        f'file:/etc/gcloud/service_account.json'
+    ],
+}
+
+actions['gcloud_select_project'] = {
+    'command': f"gcloud config set project '{project}'",
+    'unless': f"gcloud config get-value project | grep -q '^{project}$'",
+    'needs': [
+        f'action:gcloud_activate_service_account'
+    ],
+}

bundles/gcloud/metadata.py

@@ -0,0 +1,14 @@
+defaults = {
+    'apt': {
+        'packages': {
+            'apt-transport-https': {},
+            'ca-certificates': {},
+            'gnupg': {},
+            'google-cloud-sdk': {},
+            'python3-crcmod': {},
+        },
+        'sources': [
+            'deb https://packages.cloud.google.com/apt cloud-sdk main',
+        ],
+    },
+}

bundles/gitea/metadata.py

@@ -51,12 +51,19 @@ defaults = {
                         'WantedBy': 'multi-user.target',
                     },
                 },
-                'needs': {
+                'needs': [
                     'action:chmod_gitea',
                     'download:/usr/local/bin/gitea',
                     'file:/etc/systemd/system/gitea.service',
                     'file:/etc/gitea/app.ini',
-                },
+                ],
+            },
+        },
+    },
+    'zfs': {
+        'datasets': {
+            'tank/gitea': {
+                'mountpoint': '/var/lib/gitea',
             },
         },
     },

bundles/gocryptfs-inspect/items.py

@@ -0,0 +1,6 @@
+directories['/opt/gocryptfs-inspect'] = {}
+
+git_deploy['/opt/gocryptfs-inspect'] = {
+    'repo': 'https://github.com/slackner/gocryptfs-inspect.git',
+    'rev': 'ecd296c8f014bf18f5889e3cb9cb64807ff6b9c4',
+}

bundles/gocryptfs-inspect/metadata.py

@@ -0,0 +1,7 @@
+defaults = {
+    'apt': {
+        'packages': {
+            'python3-pycryptodome': {},
+        },
+    },
+}

bundles/gocryptfs/items.py

@@ -0,0 +1,43 @@
+from json import dumps
+
+directories['/etc/gocryptfs'] = {
+    'purge': True,
+}
+
+files['/etc/gocryptfs/masterkey'] = {
+    'content': node.metadata.get('gocryptfs/masterkey'),
+    'mode': '500',
+}
+
+files['/etc/gocryptfs/gocryptfs.conf'] = {
+    'content': dumps({
+    	'Version': 2,
+    	'Creator': 'gocryptfs 1.6.1',
+    	'ScryptObject': {
+    		'Salt': node.metadata.get('gocryptfs/salt'),
+    		'N': 65536,
+    		'R': 8,
+    		'P': 1,
+    		'KeyLen': 32,
+    	},
+    	'FeatureFlags': [
+    		'GCMIV128',
+    		'HKDF',
+    		'PlaintextNames',
+    		'AESSIV',
+    	]
+    }, indent=4, sort_keys=True)
+}
+
+for path, options in node.metadata.get('gocryptfs/paths').items():
+    directories[options['mountpoint']] = {
+        'owner': None,
+        'group': None,
+        'mode': None,
+        'preceded_by': [
+            f'svc_systemd:gocryptfs-{options["id"]}:stop',
+        ],
+        'needed_by': [
+            f'svc_systemd:gocryptfs-{options["id"]}',
+        ],
+    }

bundles/gocryptfs/metadata.py

@@ -0,0 +1,103 @@
+from hashlib import sha3_256
+from base64 import b64decode, b64encode
+from binascii import hexlify
+from uuid import UUID
+
+defaults = {
+    'apt': {
+        'packages': {
+            'gocryptfs': {},
+            'fuse': {},
+            'socat': {},
+        },
+    },
+    'gocryptfs': {
+        'paths': {},
+    },
+}
+
+
+@metadata_reactor.provides(
+    'gocryptfs',
+)
+def config(metadata):
+    return {
+        'gocryptfs': {
+            'masterkey': hexlify(b64decode(
+                str(repo.vault.random_bytes_as_base64_for(metadata.get('id'), length=32))
+            )).decode(),
+            'salt': b64encode(
+                sha3_256(UUID(metadata.get('id')).bytes).digest()
+            ).decode(),
+        },
+    }
+
+
+@metadata_reactor.provides(
+    'gocryptfs',
+)
+def paths(metadata):
+    paths = {}
+    
+    for path, options in metadata.get('gocryptfs/paths').items():
+        paths[path] = {
+            'id': hexlify(sha3_256(path.encode()).digest()[:8]).decode(),
+        }
+    
+    return {
+        'gocryptfs': {
+            'paths': paths,
+        },
+    }
+
+
+
+@metadata_reactor.provides(
+    'systemd/services',
+)
+def systemd(metadata):
+    services = {}
+    
+    for path, options in metadata.get('gocryptfs/paths').items():
+        services[f'gocryptfs-{options["id"]}'] = {
+            'content': {
+                'Unit': {
+                    'Description': f'gocryptfs@{path} ({options["id"]})',
+                    'After': {
+                      'filesystem.target',
+                      'zfs.target',
+                    },
+                },
+                'Service': {
+                    'RuntimeDirectory': 'gocryptfs',
+                    'Environment': {
+                        'MASTERKEY': metadata.get('gocryptfs/masterkey'),
+                        'SOCKET': f'/var/run/gocryptfs/{options["id"]}',
+                        'PLAIN': path,
+                        'CIPHER': options["mountpoint"]
+                    },
+                    'ExecStart': [
+                        '/usr/bin/gocryptfs -fg -plaintextnames -reverse -masterkey $MASTERKEY -ctlsock $SOCKET $PLAIN $CIPHER',
+                    ],
+                    'ExecStopPost': [
+                        '/usr/bin/umount $CIPHER'
+                    ],
+                },
+            },
+            'needs': [
+                'pkg_apt:gocryptfs',
+                'pkg_apt:fuse',
+                'pkg_apt:socat',
+                'file:/etc/gocryptfs/masterkey',
+                'file:/etc/gocryptfs/gocryptfs.conf',
+            ],
+            'triggers': [
+                f'svc_systemd:gocryptfs-{options["id"]}:restart',
+            ],
+        }
+
+    return {
+        'systemd': {
+            'services': services,
+        },
+    }

bundles/hetzner-cloud/metadata.py

@@ -0,0 +1,8 @@
+# defaults = {
+#     'network': {
+#         'external': {
+#             'gateway4': '172.31.1.1',
+#             'gateway6': 'fe80::1',
+#         },
+#     }, 
+# }

bundles/influxdb2/items.py

@@ -0,0 +1,2 @@
+#sudo systemctl unmask influxdb.service
+#sudo systemctl start influxdb

bundles/influxdb2/metadata.py

@@ -0,0 +1,10 @@
+defaults = {
+    'apt': {
+        'packages': {
+            'influxdb2': {},
+        },
+        'sources': [
+            'deb https://repos.influxdata.com/debian {release} stable',
+        ],
+    },
+}

bundles/l4d2/metadata.py

@@ -2,6 +2,6 @@
 def steam(metadata):
     return {
         'steam': {
-            222860: 'l4d2',
+            '222860': 'l4d2',
         },
     }

bundles/mailserver/items.py

@@ -1,7 +1,10 @@
 assert node.has_bundle('postfix')
+assert node.has_bundle('opendkim')
 assert node.has_bundle('dovecot')
 assert node.has_bundle('letsencrypt')
 assert node.has_bundle('roundcube')
+assert node.has_bundle('rspamd')
+assert node.has_bundle('redis')
 
 from hashlib import md5
 from shlex import quote

bundles/mailserver/metadata.py

@@ -1,15 +1,18 @@
+from ipaddress import ip_interface
+
 database_password = repo.vault.password_for(f'{node.name} db mailserver')
 
 defaults = {
     'mailserver': {
         'maildir': '/var/vmail',
         'database': {
-            'host': 'localhost',
+            'host': '127.0.0.1', # dont use localhost
             'name': 'mailserver',
             'user': 'mailserver',
             'password': database_password,
         },
         'test_password': repo.vault.password_for(f'{node.name} test_pw mailserver'),
+        'domains': [],
     },
     'postgresql': {
         'roles': {
@@ -33,6 +36,23 @@ defaults = {
     },
 }
 
+
+@metadata_reactor.provides(
+    'dns',
+)
+def dns(metadata):
+    dns = {}
+    
+    for domain in metadata.get('mailserver/domains'):
+        dns[domain] = {
+            'MX': [f'5 {domain}.'],
+            'TXT': ['v=spf1 a mx -all'],
+        }
+
+    return {
+        'dns': dns,
+    }
+
 @metadata_reactor.provides(
     'letsencrypt/domains',
 )

bundles/network/metadata.py

@@ -0,0 +1,46 @@
+from ipaddress import ip_interface
+
+defaults = {
+    'network': {
+    }
+}
+
+
+@metadata_reactor.provides(
+    'systemd-networkd/networks',
+)
+def systemd_networkd(metadata):
+    units = {}
+    
+    for type, network in metadata.get('network').items():
+        units[type] = {
+            'Match': {
+                'Name': network['interface'],
+            },
+            'Network': {
+                'DHCP': 'no',
+                'IPv6AcceptRA': 'no',
+            }
+        }
+        
+        for i in [4, 6]:
+            if network.get(f'ipv{i}', None):
+                units[type].update({
+                    f'Address#ipv{i}': {
+                        'Address': network[f'ipv{i}'],
+                    },
+                })
+                if f'gateway{i}' in network:
+                    units[type].update({
+                        f'Route#ipv{i}': {
+                            'Gateway': network[f'gateway{i}'],
+                            'GatewayOnlink': 'yes',
+                        }
+                    })
+
+    
+    return {
+        'systemd-networkd': {
+            'networks': units,
+        }
+    }

bundles/nextcloud/files/managed.config.php

@@ -0,0 +1,19 @@
+<?php
+# https://docs.nextcloud.com/server/stable/admin_manual/configuration_server/config_sample_php_parameters.html#multiple-config-php-file
+$CONFIG = array (
+  'dbuser' => 'nextcloud',
+  'dbpassword' => '${db_password}',
+  'dbname' => 'nextcloud',
+  'dbhost' => 'localhost',
+  'dbtype' => 'pgsql',
+  'datadirectory' => '/var/lib/nextcloud',
+  'dbport' => '5432',
+  'apps_paths' => [
+    [
+      'path'=> '/var/lib/nextcloud/.apps',
+      'url' => '/userapps',
+      'writable' => true,
+    ],
+  ],
+  'cache_path' => '/var/lib/nextcloud/.cache',
+);

bundles/nextcloud/items.py

@@ -0,0 +1,145 @@
+assert node.has_bundle('php')
+
+from shlex import quote
+from os.path import join
+from mako.template import Template
+
+def occ(command, *args, **kwargs):
+    return f"""sudo -u www-data php /opt/nextcloud/occ {command} {' '.join(args)} {' '.join(f'--{name.replace("_", "-")}' + (f'={value}' if value else '') for name, value in kwargs.items())}"""
+
+version = node.metadata.get('nextcloud/version')
+
+# DOWNLOAD
+
+downloads[f'/tmp/nextcloud-{version}.tar.bz2'] = {
+    'url': f'https://download.nextcloud.com/server/releases/nextcloud-{version}.tar.bz2',
+    'sha256': node.metadata.get('nextcloud/sha256'),
+    'triggered': True,
+}
+actions['delete_nextcloud'] = {
+    'command': 'rm -rf /opt/nextcloud/*',
+    'triggered': True,
+}
+actions['extract_nextcloud'] = {
+    'command': f'tar xfvj /tmp/nextcloud-{version}.tar.bz2 --skip-old-files --strip 1 -C /opt/nextcloud nextcloud',
+    'unless': f"""php -r 'include "/opt/nextcloud/version.php"; echo "$OC_VersionString";' | grep -q '^{version}$'""",
+    'preceded_by': [
+        'action:delete_nextcloud',
+        f'download:/tmp/nextcloud-{version}.tar.bz2',
+    ],
+    'needs': [
+        'action:symlink_/opt/nextcloud/config',
+        'directory:/opt/nextcloud',
+    ],
+}
+
+# DIRECTORIES, FILES AND SYMLINKS
+
+directories['/etc/nextcloud'] = {
+    'owner': 'www-data',
+    'group': 'www-data',
+}
+directories['/opt/nextcloud'] = {}
+directories['/var/lib/nextcloud'] = {
+    'owner': 'www-data',
+    'group': 'www-data',
+    'mode': '770',
+}
+directories['/var/lib/nextcloud/.apps'] = {
+    'owner': 'www-data',
+    'group': 'www-data',
+}
+directories['/var/lib/nextcloud/.cache'] = {
+    'owner': 'www-data',
+    'group': 'www-data',
+}
+files['/etc/nextcloud/CAN_INSTALL'] = {
+    'content': '',
+    'owner': 'www-data',
+    'group': 'www-data',
+    'mode': '640',
+    'needs': [
+        'directory:/etc/nextcloud',
+    ],
+}
+files['/etc/nextcloud/managed.config.php'] = {
+    'content_type': 'mako',
+    'owner': 'www-data',
+    'group': 'www-data',
+    'mode': '640',
+    'context': {
+        'db_password': node.metadata.get('postgresql/roles/nextcloud/password'),
+    },
+    'needs': [
+        'directory:/etc/nextcloud',
+    ],
+}
+actions['symlink_/opt/nextcloud/config'] = {
+    'command': f'ln -s /etc/nextcloud /opt/nextcloud/config && chown www-data:www-data /opt/nextcloud/config',
+    'unless': 'readlink /opt/nextcloud/config | grep -q /etc/nextcloud',
+    'needs': [
+        'action:delete_nextcloud',
+        'directory:/etc/nextcloud',
+    ],
+}
+actions['symlink_/opt/nextcloud/userapps'] = {
+    'command': f'ln -s /var/lib/nextcloud/.apps /opt/nextcloud/userapps && chown www-data:www-data /opt/nextcloud/userapps',
+    'unless': 'readlink /opt/nextcloud/userapps | grep -q /var/lib/nextcloud/.apps',
+    'needs': [
+        'action:delete_nextcloud',
+        'directory:/var/lib/nextcloud/.apps',
+    ],
+}
+
+# SETUP
+
+actions['install_nextcloud'] = {
+    'command': occ(
+        'maintenance:install',
+        no_interaction=None,
+        database='pgsql',
+        database_name='nextcloud',
+        database_host='localhost',
+        database_user='nextcloud',
+        database_pass=node.metadata.get('postgresql/roles/nextcloud/password'),
+        admin_user='admin',
+        admin_pass=node.metadata.get('nextcloud/admin_pass'),
+        data_dir='/var/lib/nextcloud',
+    ),
+    'unless': occ('status') + ' | grep -q "installed: true"',
+    'needs': [
+        'directory:/etc/nextcloud',
+        'directory:/opt/nextcloud',
+        'directory:/var/lib/nextcloud',
+        'directory:/var/lib/nextcloud/.apps',
+        'directory:/var/lib/nextcloud/.cache',
+        'file:/etc/nextcloud/CAN_INSTALL',
+        'file:/etc/nextcloud/managed.config.php',
+        'action:extract_nextcloud',
+        'action:symlink_/opt/nextcloud/userapps',
+        'action:symlink_/opt/nextcloud/config',
+        'postgres_db:nextcloud',
+    ],
+}
+
+# UPGRADE
+
+actions['upgrade_nextcloud'] = {
+    'command': occ('upgrade'),
+    'unless': occ('status') + f' | grep -q "versionstring: {version}"',
+    'needs': [
+        'action:install_nextcloud',
+    ],
+}
+
+actions['nextcloud_add_missing_inidces'] = {
+    'command': occ('db:add-missing-indices'),
+    'needs': [
+        'action:upgrade_nextcloud',
+    ],
+    'triggered': True,
+    'triggered_by': [
+        f'action:extract_nextcloud',
+        f'action:upgrade_nextcloud',
+    ],
+}

bundles/nextcloud/metadata.py

@@ -0,0 +1,72 @@
+import string
+from uuid import UUID
+
+defaults = {
+    'apt': {
+        'packages': {
+            'php': {},
+            'php-curl': {},
+            'php-gd': {},
+            'php-json': {},
+            'php-xml': {},
+            'php-mbstring': {},
+            'php-cli': {},
+            'php-cgi': {},
+            'php-zip': {},
+        },
+    },
+    'archive': {
+        'paths': {
+            '/var/lib/nextcloud': {
+                'exclude': [
+                    '^appdata_',
+                    '^updater-',
+                    '^nextcloud\.log',
+                    '^updater\.log',
+                    '^[^/]+/cache',
+                    '^[^/]+/files_versions',
+                    '^[^/]+/files_trashbin',
+                ],
+            },
+        },
+    },
+    'backup': {
+        'paths': [
+            '/etc/nextcloud/config.php',
+        ],
+    },
+    'nextcloud': {
+        'admin_user': 'admin',
+        'admin_pass': repo.vault.password_for(f'{node.name} nextcloud admin pw'),
+    },
+    'nginx': {
+        'vhosts': {
+            'nextcloud': {
+                'webroot': '/opt/nextcloud',
+                'php': True,
+            },
+        },
+    },
+    'postgresql': {
+        'roles': {
+            'nextcloud': {
+                'password': repo.vault.password_for(f'{node.name} nextcloud db pw'),
+            },
+        },
+        'databases': {
+            'nextcloud': {
+                'owner': 'nextcloud',
+            },
+        },
+    },
+    'zfs': {
+        'datasets': {
+            'tank/nextcloud': {
+                'mountpoint': '/var/lib/nextcloud',
+                'needed_by': [
+                    'bundle:nextcloud',
+                ],
+            },
+        },
+    },
+}

bundles/nginx/files/site_template

@@ -51,9 +51,6 @@ server {
     add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";
 % endif
 
-    resolver 8.8.8.8 8.8.4.4 valid=300s;
-    resolver_timeout 5s;
-
     access_log /var/log/nginx/access-${vhost}.log;
     error_log /var/log/nginx/error-${vhost}.log;
 

bundles/nginx/items.py

@@ -69,6 +69,7 @@ for vhost, config in node.metadata.get('nginx/vhosts', {}).items():
             'create_access_log': config.get('access_log', node.metadata.get('nginx/access_log', False)),
             'php_version': node.metadata.get('php/version', ''),
             'vhost': vhost,
+            'nameservers': node.metadata.get('nameservers'),
             **config,
         },
         'needs': set(),

bundles/nginx/metadata.py

@@ -1,4 +1,4 @@
-from bundlewrap.metadata import atomic
+from ipaddress import ip_interface
 
 defaults = {
     'apt': {
@@ -12,6 +12,30 @@ defaults = {
 }
 
 
+@metadata_reactor.provides(
+    'dns',
+)
+def dns(metadata):
+    dns = {}
+    
+    for config in metadata.get('nginx/vhosts', {}).values():
+        dns[config['domain']] = {
+            'A': [
+                str(ip_interface(network['ipv4']).ip)
+                    for network in metadata.get('network').values()
+                    if 'ipv4' in network
+            ],
+            'AAAA': [
+                str(ip_interface(network['ipv6']).ip)
+                    for network in metadata.get('network').values()
+                    if 'ipv6' in network
+            ],
+        }
+
+    return {
+        'dns': dns,
+    }
+
 @metadata_reactor.provides(
     'letsencrypt/domains',
     'letsencrypt/reload_after',

bundles/opendkim/files/key_table

@@ -0,0 +1,3 @@
+% for domain in domains:
+mail._domainkey.${domain} ${domain}:mail:/etc/opendkim/keys/${domain}/mail.private
+% endfor

bundles/opendkim/files/opendkim.conf

@@ -0,0 +1,15 @@
+Mode                    sv
+SignatureAlgorithm      rsa-sha256
+Canonicalization        relaxed/simple
+KeyTable                refile:/etc/opendkim/key_table
+SigningTable            refile:/etc/opendkim/signing_table
+
+UMask                   007
+UserID                  opendkim:opendkim
+PidFile                 /run/opendkim/opendkim.pid
+Socket                  inet:8891@localhost
+
+Syslog                  yes
+SyslogSuccess           Yes
+SyslogFacility          mail
+LogWhy                  Yes

bundles/opendkim/files/signing_table

@@ -0,0 +1,3 @@
+% for domain in domains:
+*@${domain} mail._domainkey.${domain}
+% endfor

bundles/opendkim/items.py

@@ -0,0 +1,86 @@
+file_attributes = {
+    'owner': 'opendkim',
+    'group': 'opendkim',
+    'mode': '700',
+    'triggers': [
+        'svc_systemd:opendkim:restart',
+    ],
+}
+
+groups['opendkim'] = {}
+users['opendkim'] = {}
+
+directories = {
+    '/etc/opendkim': {
+        **file_attributes,
+        'purge' : True,
+    },
+    '/etc/opendkim/keys': {
+        **file_attributes,
+        'purge' : True,
+    },
+}
+
+files = {
+    '/etc/opendkim.conf': {
+        **file_attributes,
+    },
+    '/etc/defaults/opendkim': {
+        # https://metadata.ftp-master.debian.org/changelogs//main/o/opendkim/testing_opendkim.NEWS
+        'delete': True,
+    },
+    '/etc/opendkim/key_table': {
+        'content_type': 'mako',
+        'context': {
+            'domains': node.metadata.get('mailserver/domains'),
+        },
+        **file_attributes,
+    },
+    '/etc/opendkim/signing_table': {
+        'content_type': 'mako',
+        'context': {
+            'domains': node.metadata.get('mailserver/domains'),
+        },
+        **file_attributes,
+    },
+}
+
+for domain in node.metadata.get('mailserver/domains'):
+    directories[f'/etc/opendkim/keys/{domain}'] = {
+        **file_attributes,
+        'purge': True,
+    }
+    files[f'/etc/opendkim/keys/{domain}/mail.private'] = {
+        **file_attributes,
+        'content': node.metadata.get(f'opendkim/keys/{domain}/private'),
+    }
+    # files[f'/etc/opendkim/keys/{domain}/mail.txt'] = {
+    #     **file_attributes,
+    #     'content_type': 'any',
+    # }
+    # actions[f'generate_{domain}_dkim_key'] = {
+    #     'command': (
+    #         f'sudo --user opendkim'
+    #         f' opendkim-genkey'
+    #         f' --selector=mail'
+    #         f' --directory=/etc/opendkim/keys/{domain}'
+    #         f' --domain={domain}'
+    #     ),
+    #     'unless': f'test -f /etc/opendkim/keys/{domain}/mail.private',
+    #     'needs': [
+    #         'svc_systemd:opendkim',
+    #         f'directory:/etc/opendkim/keys/{domain}',
+    #     ],
+    #     'triggers': [
+    #         'svc_systemd:opendkim:restart',
+    #     ],
+    # }
+
+svc_systemd['opendkim'] = {
+    'needs': [
+        'pkg_apt:opendkim',
+        'file:/etc/opendkim.conf',
+        'file:/etc/opendkim/key_table',
+        'file:/etc/opendkim/signing_table',
+    ],
+}

bundles/opendkim/metadata.py

@@ -0,0 +1,93 @@
+from os.path import join, exists
+from re import sub
+from cryptography.hazmat.primitives import serialization as crypto_serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.hazmat.backends import default_backend as crypto_default_backend
+
+
+defaults = {
+    'apt': {
+        'packages': {
+            'opendkim': {},
+            'opendkim-tools': {},
+        },
+    },
+    'opendkim': {
+        'keys': {},
+    },
+    'dns': {
+        'mail._domainkey.mail2.sublimity.de': {
+            'TXT': [
+                
+            ]
+        }
+    }
+}
+
+
+@metadata_reactor.provides(
+    'opendkim/keys',
+)
+def keys(metadata):
+    keys = {}
+    
+    for domain in metadata.get('mailserver/domains'):
+        if domain in metadata.get(f'opendkim/keys'):
+            continue
+        
+        pubkey_path = join(repo.path, 'data', 'dkim', f'{domain}.pubkey')
+        privkey_path = join(repo.path, 'data', 'dkim', f'{domain}.privkey.enc')
+        
+        if not exists(pubkey_path) or not exists(privkey_path):
+            key = rsa.generate_private_key(
+                backend=crypto_default_backend(),
+                public_exponent=65537,
+                key_size=2048
+            )
+            with open(pubkey_path, 'w') as file:
+                file.write(     
+                    key.public_key().public_bytes(
+                        crypto_serialization.Encoding.OpenSSH,
+                        crypto_serialization.PublicFormat.OpenSSH
+                    ).decode()
+                )
+            with open(privkey_path, 'w') as file:
+                file.write(
+                    repo.vault.encrypt(
+                        key.private_bytes(
+                            crypto_serialization.Encoding.PEM,
+                            crypto_serialization.PrivateFormat.PKCS8,
+                            crypto_serialization.NoEncryption()
+                        ).decode(),
+                    )
+                )
+                
+        with open(pubkey_path, 'r') as pubkey:
+            with open(privkey_path, 'r') as privkey:
+                keys[domain] = {
+                    'public': pubkey.read(),
+                    'private': repo.vault.decrypt(privkey.read()),
+                }
+
+    return {
+        'opendkim': {
+            'keys': keys,
+        }
+    }
+
+
+@metadata_reactor.provides(
+    'dns',
+)
+def dns(metadata):
+    dns = {}
+    
+    for domain, keys in metadata.get('opendkim/keys').items():
+        raw_key = sub('^ssh-rsa ', '', keys['public'])
+        dns[f'mail._domainkey.{domain}'] = {
+            'TXT': [f'v=DKIM1; k=rsa; p={raw_key}'],
+        }
+    
+    return {
+        'dns': dns,
+    }

bundles/php/items.py

@@ -1,7 +1,7 @@
 version = node.metadata.get('php/version')
 
 php_ini_context = {
-    'num_cpus': node.metadata.get('vm/cpu'),
+    'num_cpus': node.metadata.get('vm/cores'),
     'post_max_size': node.metadata.get('php/post_max_size', 10),
 }
 

bundles/postfix/files/main.cf

@@ -1,63 +1,53 @@
-# See /usr/share/postfix/main.cf.dist for a commented, more complete version
-
-
-# Debian specific:  Specifying a file name will cause the first
-# line of that file to be used as the name.  The Debian default
-# is /etc/mailname.
-#myorigin = /etc/mailname
-
 smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU)
 biff = no
-
-# appending .domain is the MUA's job.
 append_dot_mydomain = no
-
-# Uncomment the next line to generate "delayed mail" warnings
-#delay_warning_time = 4h
-
 readme_directory = no
-
-# TLS parameters
-smtpd_tls_cert_file = /etc/letsencrypt/live/mail.sublimity.de/fullchain.pem
-smtpd_tls_key_file = /etc/letsencrypt/live/mail.sublimity.de/privkey.pem
+compatibility_level = 2
 smtpd_use_tls=yes
 <%text>
 smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache
 smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
 </%text>
-
-# See /usr/share/doc/postfix/TLS_README.gz in the postfix-doc package for
-# information on enabling SSL in the smtp client.
-
 smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination
-myhostname = mail.sublimity.de
+myhostname = debian-2gb-nbg1-1
 alias_maps = hash:/etc/aliases
 alias_database = hash:/etc/aliases
 myorigin = /etc/mailname
-mydestination = mail.sublimity.de, localhost.localdomain, localhost
+mydestination = $myhostname, debian-2gb-nbg1-1, localhost.localdomain, localhost
 relayhost =
 mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
 mailbox_size_limit = 0
 recipient_delimiter = +
 inet_interfaces = all
 inet_protocols = all
-virtual_mailbox_domains = psql:/etc/postfix/virtual_domains.cf
-virtual_mailbox_maps = psql:/etc/postfix/virtual_mailboxes.cf
-virtual_alias_maps = psql:/etc/postfix/virtual_aliases.cf,psql:/etc/postfix/virtual_mailboxes.cf
+
+virtual_mailbox_domains = pgsql:/etc/postfix/virtual_mailbox_domains.cf
+virtual_mailbox_maps = pgsql:/etc/postfix/virtual_mailbox_maps.cf
+virtual_alias_maps = pgsql:/etc/postfix/virtual_alias_maps.cf,pgsql:/etc/postfix/virtual_mailbox_maps.cf
+smtpd_sender_login_maps = pgsql:/etc/postfix/virtual_alias_maps.cf
 virtual_transport = lmtp:unix:private/dovecot-lmtp
+
 smtpd_sasl_type = dovecot
 smtpd_sasl_path = private/auth
 smtpd_sasl_auth_enable = yes
+
 smtpd_tls_security_level = may
 smtpd_tls_auth_only = yes
-message_size_limit = 1000000000
-smtpd_milters = unix:/spamass/spamass.sock
-milter_connect_macros = i j {daemon_name} v {if_name} _
-policyd-spf_time_limit = 3600
+smtpd_tls_cert_file = /var/lib/dehydrated/certs/${node.metadata.get('mailserver/hostname')}/fullchain.pem
+smtpd_tls_key_file = /var/lib/dehydrated/certs/${node.metadata.get('mailserver/hostname')}/privkey.pem
+smtp_tls_security_level = may
 
-smtpd_recipient_restrictions = permit_mynetworks,reject_unauth_destination,check_policy_service unix:private/policyd-spf
-#  reject_rbl_client dnsbl.sorbs.net
-#  reject_rbl_client bl.spamcop.net
-#  reject_rbl_client zen.spamhaus.org
-#  reject_rbl_client dnsbl1.uceprotect.net
-#  reject_unauth_destination
+smtpd_restriction_classes = mua_sender_restrictions, mua_client_restrictions, mua_helo_restrictions
+mua_client_restrictions = permit_sasl_authenticated, reject
+mua_sender_restrictions = permit_sasl_authenticated, reject
+mua_helo_restrictions = permit_mynetworks, reject_non_fqdn_hostname, reject_invalid_hostname, permit
+
+smtpd_milters = inet:localhost:8891 inet:127.0.0.1:11332
+non_smtpd_milters = inet:localhost:8891 inet:127.0.0.1:11332
+
+# opendkim
+milter_protocol = 6
+milter_default_action = accept
+
+# rspamd
+milter_mail_macros = "i {mail_addr} {client_addr} {client_name} {auth_authen}"

bundles/postfix/files/master.cf

@@ -1,124 +1,37 @@
-#
-# Postfix master process configuration file.  For details on the format
-# of the file, see the master(5) manual page (command: "man 5 master" or
-# on-line: http://www.postfix.org/master.5.html).
-#
-# Do not forget to execute "postfix reload" after editing this file.
-#
 # ==========================================================================
 # service type  private unpriv  chroot  wakeup  maxproc command + args
-#               (yes)   (yes)   (yes)   (never) (100)
+#               (yes)   (yes)   (no)    (never) (100)
 # ==========================================================================
-smtp      inet  n       -       -       -       -       smtpd
-#smtp      inet  n       -       -       -       1       postscreen
-#smtpd     pass  -       -       -       -       -       smtpd
-#dnsblog   unix  -       -       -       -       0       dnsblog
-#tlsproxy  unix  -       -       -       -       0       tlsproxy
-#submission inet n       -       -       -       -       smtpd
-#  -o syslog_name=postfix/submission
-#  -o smtpd_tls_security_level=encrypt
-#  -o smtpd_sasl_auth_enable=yes
-#  -o smtpd_reject_unlisted_recipient=no
-#  -o smtpd_client_restrictions=$mua_client_restrictions
-#  -o smtpd_helo_restrictions=$mua_helo_restrictions
-#  -o smtpd_sender_restrictions=$mua_sender_restrictions
-#  -o smtpd_recipient_restrictions=
-#  -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
-#  -o milter_macro_daemon_name=ORIGINATING
-
-submission inet n       -       -       -       -       smtpd
-  -o syslog_name=postfix/submission
-  -o smtpd_tls_security_level=encrypt
-  -o smtpd_sasl_auth_enable=yes
-  -o smtpd_client_restrictions=permit_sasl_authenticated,reject
-
-#smtps     inet  n       -       -       -       -       smtpd
-#  -o syslog_name=postfix/smtps
-#  -o smtpd_tls_wrappermode=yes
-#  -o smtpd_sasl_auth_enable=yes
-#  -o smtpd_reject_unlisted_recipient=no
-#  -o smtpd_client_restrictions=$mua_client_restrictions
-#  -o smtpd_helo_restrictions=$mua_helo_restrictions
-#  -o smtpd_sender_restrictions=$mua_sender_restrictions
-#  -o smtpd_recipient_restrictions=
-#  -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
-#  -o milter_macro_daemon_name=ORIGINATING
-#628       inet  n       -       -       -       -       qmqpd
-pickup    unix  n       -       -       60      1       pickup
-cleanup   unix  n       -       -       -       0       cleanup
+smtp      inet  n       -       y       -       -       smtpd
+pickup    unix  n       -       y       60      1       pickup
+cleanup   unix  n       -       y       -       0       cleanup
 qmgr      unix  n       -       n       300     1       qmgr
-#qmgr     unix  n       -       n       300     1       oqmgr
-tlsmgr    unix  -       -       -       1000?   1       tlsmgr
-rewrite   unix  -       -       -       -       -       trivial-rewrite
-bounce    unix  -       -       -       -       0       bounce
-defer     unix  -       -       -       -       0       bounce
-trace     unix  -       -       -       -       0       bounce
-verify    unix  -       -       -       -       1       verify
-flush     unix  n       -       -       1000?   0       flush
+tlsmgr    unix  -       -       y       1000?   1       tlsmgr
+rewrite   unix  -       -       y       -       -       trivial-rewrite
+bounce    unix  -       -       y       -       0       bounce
+defer     unix  -       -       y       -       0       bounce
+trace     unix  -       -       y       -       0       bounce
+verify    unix  -       -       y       -       1       verify
+flush     unix  n       -       y       1000?   0       flush
 proxymap  unix  -       -       n       -       -       proxymap
 proxywrite unix -       -       n       -       1       proxymap
-smtp      unix  -       -       -       -       -       smtp
-relay     unix  -       -       -       -       -       smtp
-#       -o smtp_helo_timeout=5 -o smtp_connect_timeout=5
-showq     unix  n       -       -       -       -       showq
-error     unix  -       -       -       -       -       error
-retry     unix  -       -       -       -       -       error
-discard   unix  -       -       -       -       -       discard
+smtp      unix  -       -       y       -       -       smtp
+relay     unix  -       -       y       -       -       smtp
+        -o syslog_name=postfix/$service_name
+showq     unix  n       -       y       -       -       showq
+error     unix  -       -       y       -       -       error
+retry     unix  -       -       y       -       -       error
+discard   unix  -       -       y       -       -       discard
 local     unix  -       n       n       -       -       local
 virtual   unix  -       n       n       -       -       virtual
-lmtp      unix  -       -       -       -       -       lmtp
-anvil     unix  -       -       -       -       1       anvil
-scache    unix  -       -       -       -       1       scache
-#
-# ====================================================================
-# Interfaces to non-Postfix software. Be sure to examine the manual
-# pages of the non-Postfix software to find out what options it wants.
-#
-# Many of the following services use the Postfix pipe(8) delivery
-# agent.  See the pipe(8) man page for information about ${recipient}
-# and other message envelope options.
-# ====================================================================
-#
-# maildrop. See the Postfix MAILDROP_README file for details.
-# Also specify in main.cf: maildrop_destination_recipient_limit=1
-#
+lmtp      unix  -       -       y       -       -       lmtp
+anvil     unix  -       -       y       -       1       anvil
+scache    unix  -       -       y       -       1       scache
+postlog   unix-dgram n  -       n       -       1       postlogd
 maildrop  unix  -       n       n       -       -       pipe
   flags=DRhu user=vmail argv=/usr/bin/maildrop -d ${recipient}
-#
-# ====================================================================
-#
-# Recent Cyrus versions can use the existing "lmtp" master.cf entry.
-#
-# Specify in cyrus.conf:
-#   lmtp    cmd="lmtpd -a" listen="localhost:lmtp" proto=tcp4
-#
-# Specify in main.cf one or more of the following:
-#  mailbox_transport = lmtp:inet:localhost
-#  virtual_transport = lmtp:inet:localhost
-#
-# ====================================================================
-#
-# Cyrus 2.1.5 (Amos Gouaux)
-# Also specify in main.cf: cyrus_destination_recipient_limit=1
-#
-#cyrus     unix  -       n       n       -       -       pipe
-#  user=cyrus argv=/cyrus/bin/deliver -e -r ${sender} -m ${extension} ${user}
-#
-# ====================================================================
-# Old example of delivery via Cyrus.
-#
-#old-cyrus unix  -       n       n       -       -       pipe
-#  flags=R user=cyrus argv=/cyrus/bin/deliver -e -m ${extension} ${user}
-#
-# ====================================================================
-#
-# See the Postfix UUCP_README file for configuration details.
-#
 uucp      unix  -       n       n       -       -       pipe
   flags=Fqhu user=uucp argv=uux -r -n -z -a$sender - $nexthop!rmail ($recipient)
-#
-# Other external delivery methods.
-#
 ifmail    unix  -       n       n       -       -       pipe
   flags=F user=ftn argv=/usr/lib/ifmail/ifmail -r $nexthop ($recipient)
 bsmtp     unix  -       n       n       -       -       pipe
@@ -128,5 +41,15 @@ scalemail-backend unix	-	n	n	-	2	pipe
 mailman   unix  -       n       n       -       -       pipe
   flags=FR user=list argv=/usr/lib/mailman/bin/postfix-to-mailman.py
   ${nexthop} ${user}
-policyd-spf  unix  -       n       n       -       0       spawn
-    user=policyd-spf argv=/usr/bin/policyd-spf
+submission inet n       -       y       -       -       smtpd
+   -o syslog_name=postfix/submission
+   -o smtpd_tls_security_level=encrypt
+   -o smtpd_sasl_auth_enable=yes
+   -o smtpd_tls_auth_only=yes
+   -o smtpd_reject_unlisted_recipient=no
+   -o smtpd_client_restrictions=$mua_client_restrictions
+   -o smtpd_helo_restrictions=$mua_helo_restrictions
+   -o smtpd_sender_restrictions=$mua_sender_restrictions
+   -o smtpd_recipient_restrictions=
+   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
+   -o milter_macro_daemon_name=ORIGINATING

bundles/postfix/files/virtual_alias_maps.cf

@@ -2,4 +2,4 @@ hosts = ${host}
 dbname = ${name}
 user = ${user}
 password = ${password}
-query = SELECT redirect FROM users LEFT JOIN domains ON users.domain_id = domains.id WHERE redirect IS NOT NULL AND users.name = SPLIT_PART('%s', '@', 1) AND domains.name = SPLIT_PART('%s', '@', 2)
+query = SELECT redirect FROM users LEFT JOIN domains ON users.domain_id = domains.id WHERE redirect IS NOT NULL AND users.name = '%u' AND domains.name = '%d'

bundles/postfix/files/virtual_mailbox_maps.cf

@@ -2,4 +2,4 @@ hosts = ${host}
 dbname = ${name}
 user = ${user}
 password = ${password}
-query = SELECT CONCAT(users.name, '@', domains.name) AS email FROM users LEFT JOIN domains ON users.domain_id = domains.id WHERE redirect IS NULL AND users.name = SPLIT_PART('%s', '@', 1) AND domains.name = SPLIT_PART('%s', '@', 2)
+query = SELECT CONCAT(users.name, '@', domains.name) AS email FROM users LEFT JOIN domains ON users.domain_id = domains.id WHERE redirect IS NULL AND users.name = '%u' AND domains.name = '%d'

bundles/postfix/items.py

@@ -40,6 +40,13 @@ svc_systemd['postfix'] = {
     ],
 }
 
+actions['test_postfix_config'] = {
+    'command': 'false',
+    'unless': "postconf check | grep -v 'symlink leaves directory' | wc -l | grep -q '^0$'",
+    'needs': [
+        'svc_systemd:postfix',
+    ],
+}
 actions['test_virtual_mailbox_domains'] = {
     'command': 'false',
     'unless': "postmap -q example.com pgsql:/etc/postfix/virtual_mailbox_domains.cf | grep -q '^example.com$'",

bundles/postfix/metadata.py

@@ -5,6 +5,11 @@ defaults = {
             'postfix-pgsql': {},
         }
     },
+    'backup': {
+        'paths': [
+            '/var/vmail',
+        ],
+    },
     'letsencrypt': {
         'reload_after': {
             'postfix',

bundles/postgresql/metadata.py

@@ -1,4 +1,14 @@
 defaults = {
+    'apt': {
+        'packages': {
+            'postgresql': {},
+        },
+    },
+    'backup': {
+        'paths': [
+            '/var/lib/postgresql',
+        ],
+    },
     'postgresql': {
         'roles': {
             'root': {
@@ -8,11 +18,6 @@ defaults = {
         },
         'databases': {},
     },
-    'apt': {
-        'packages': {
-            'postgresql': {},
-        },
-    },
 }
 
 if node.has_bundle('zfs'):

bundles/redis/metadata.py

@@ -0,0 +1,7 @@
+defaults = {
+    'apt': {
+        'packages': {
+            'redis-server': {},
+        },
+    },
+} 

bundles/rspamd/files/ip_whitelist.map

@@ -0,0 +1,3 @@
+% for ip in sorted(node.metadata.get('rspamd/ignore_spam_check_for_ips', set())):
+${ip}
+% endfor

bundles/rspamd/files/local.d/classifier-bayes.conf

@@ -0,0 +1 @@
+backend = "redis";

bundles/rspamd/files/local.d/logging.inc

@@ -0,0 +1,2 @@
+systemd = true;
+type = "console";

bundles/rspamd/files/local.d/milter_headers.conf

@@ -0,0 +1,2 @@
+use = ["x-spamd-bar", "x-spam-level", "authentication-results"];
+authenticated_headers = ["authentication-results"];

bundles/rspamd/files/local.d/multimap.conf

@@ -0,0 +1,6 @@
+IP_WHITELIST {
+    type = "ip";
+    prefilter = true;
+    map = "/etc/rspamd/local.d/ip_whitelist.map";
+    action = "accept";
+}

bundles/rspamd/files/local.d/redis.conf

@@ -0,0 +1 @@
+servers = "127.0.0.1";

bundles/rspamd/files/local.d/worker-normal.inc

@@ -0,0 +1 @@
+bind_socket = "localhost:11333";

bundles/rspamd/files/local.d/worker-proxy.inc

@@ -0,0 +1,7 @@
+bind_socket = "localhost:11332";
+milter = yes;
+timeout = 120s;
+upstream "local" {
+    default = yes;
+    self_scan = yes;
+}

bundles/rspamd/files/override.d/antivirus.conf

@@ -0,0 +1,6 @@
+clamav {
+    servers = "/run/clamav/clamd.ctl";
+    action = "reject";
+    type = "clamav";
+    symbol = "CLAM_VIRUS";
+}

bundles/rspamd/files/worker-controller.inc

@@ -0,0 +1 @@
+password = "${node.metadata.get('rspamd/web_password')}";

bundles/rspamd/items.py

@@ -0,0 +1,66 @@
+from os import listdir
+from os.path import join
+
+repo.libs.tools.require_bundle(node, 'redis', 'rspamd does not work without a redis cache')
+
+directories = {
+    '/etc/rspamd/local.d': {
+        'purge': True,
+        'needs': {
+            'pkg_apt:rspamd',
+        },
+        'triggers': {
+            'svc_systemd:rspamd:restart',
+        },
+    },
+    '/etc/rspamd/override.d': {
+        'purge': True,
+        'needs': {
+            'pkg_apt:rspamd',
+        },
+        'triggers': {
+            'svc_systemd:rspamd:restart',
+        },
+    },
+}
+
+files = {
+    '/etc/rspamd/local.d/ip_whitelist.map': {
+        'content_type': 'mako',
+        'triggers': {
+            'svc_systemd:rspamd:restart',
+        },
+    },
+    '/etc/rspamd/local.d/worker-controller.inc': {
+        'content_type': 'mako',
+        'triggers': {
+            'svc_systemd:rspamd:restart',
+        },
+    }
+}
+
+local_config_path = join(repo.path, 'bundles', 'rspamd', 'files', 'local.d')
+for f in listdir(local_config_path):
+    files[f'/etc/rspamd/local.d/{f}'] = {
+        'source': f'local.d/{f}',
+        'triggers': {
+            'svc_systemd:rspamd:restart',
+        },
+    }
+
+override_config_path = join(repo.path, 'bundles', 'rspamd', 'files', 'override.d')
+for f in listdir(override_config_path):
+    files[f'/etc/rspamd/override.d/{f}'] = {
+        'source': f'override.d/{f}',
+        'triggers': {
+            'svc_systemd:rspamd:restart',
+        },
+    }
+
+svc_systemd = {
+    'rspamd': {
+        'needs': {
+            'pkg_apt:rspamd',
+        },
+    },
+}

bundles/rspamd/metadata.py

@@ -0,0 +1,15 @@
+defaults = {
+    'apt': {
+        'packages': {
+            'clamav': {},
+            'clamav-daemon': {},
+            'clamav-freshclam': {},
+            'clamav-unofficial-sigs': {}, 
+            'rspamd': {},
+        },
+    },
+    'rspamd': {
+        'web_password': repo.vault.password_for(node.name + ' rspamd web password'),
+        'ignore_spam_check_for_ips': [],
+    },
+}

bundles/systemd-networkd/files/resolv.conf

@@ -1,3 +1,3 @@
-% for nameserver in sorted(node.metadata.get('nameservers', {'9.9.9.10', '2620:fe::10'})):
+% for nameserver in sorted(node.metadata.get('nameservers')):
 nameserver ${nameserver}
 % endfor

bundles/systemd-networkd/files/template-iface-nodhcp.network

@@ -1,50 +0,0 @@
-<%
-    from ipaddress import ip_network
-%>\
-[Match]
-Name=${interface}
-
-% for addr in sorted(config.get('ips', set())):
-[Address]
-<%
-    if '/' in addr:
-        ip, prefix = addr.split('/')
-    else:
-        ip = addr
-        prefix = '32'
-%>\
-Address=${ip}/${prefix}
-
-% endfor
-% for route, rconfig in sorted(config.get('routes', {}).items()):
-[Route]
-%   if 'via' in rconfig:
-Gateway=${rconfig['via']}
-%   endif
-Destination=${route}
-GatewayOnlink=yes
-
-% endfor
-% if 'gateway4' in config:
-[Route]
-Gateway=${config['gateway4']}
-GatewayOnlink=yes
-
-% endif
-% if 'gateway6' in config:
-[Route]
-Gateway=${config['gateway6']}
-GatewayOnlink=yes
-
-% endif
-[Network]
-DHCP=no
-IPv6AcceptRA=no
-
-% if config.get('forwarding', False):
-IPForward=yes
-%endif
-
-% for vlan in sorted(config.get('vlans', set())):
-VLAN=${interface}.${vlan}
-% endfor

bundles/systemd-networkd/items.py

@@ -1,24 +1,14 @@
 assert node.has_bundle('systemd')
 
-from bundlewrap.exceptions import BundleError
-
-
 files = {
     '/etc/network/interfaces': {
         'delete': True,
     },
 }
 
-if node.metadata.get('systemd-networkd/enable-resolved', False):
-    symlinks['/etc/resolv.conf'] = {
-        'target': '/run/systemd/resolve/stub-resolv.conf',
-    }
-    svc_systemd['systemd-resolved'] = {}
-else:
-    files['/etc/resolv.conf'] = {
-        'content_type': 'mako',
-    }
-
+files['/etc/resolv.conf'] = {
+    'content_type': 'mako',
+}
 
 directories = {
     '/etc/systemd/network': {
@@ -26,30 +16,13 @@ directories = {
     },
 }
 
-mac_host_prefix = '%04x' % (node.magic_number % 65534)
-generated_mac = f'52:54:00:{mac_host_prefix[0:2]}:{mac_host_prefix[2:4]}:{{}}'
-
-# Don't use .get() here. We might end up with a node without a network
-# config!
-for interface, config in node.metadata['interfaces'].items():
-    if config.get('dhcp', False):
-        if 'vlans' in config:
-            raise BundleError(f'{node.name} interface {interface} cannot use vlans and dhcp!')
-        template = 'template-iface-dhcp.network'
-    else:
-        template = 'template-iface-nodhcp.network'
-
-    if '.' in interface:
-        vlan_id = int(interface.split('.')[1])
-        vlan_hex = '%02x' % (vlan_id % 255)
-        files['/etc/systemd/network/60-iface-{}.netdev'.format(interface)] = {
-            'source': 'template-iface-vlan.netdev',
-            'content_type': 'mako',
-            'context': {
-                'interface': interface,
-                'vlan': vlan_id,
-                'mac': generated_mac.format(vlan_hex)
-            },
+for type, path in {
+    'networks': '/etc/systemd/network/{}.network',
+    'netdevs': '/etc/systemd/network/{}.netdev',
+}.items():
+    for name, config in node.metadata.get(f'systemd-networkd/{type}').items():
+        files[path.format(name)] = {
+            'content': repo.libs.systemd.generate_unitfile(config),
             'needed_by': {
                 'svc_systemd:systemd-networkd',
             },
@@ -57,85 +30,6 @@ for interface, config in node.metadata['interfaces'].items():
                 'svc_systemd:systemd-networkd:restart',
             },
         }
-        weight = 61
-    else:
-        weight = 50
-
-    if not config.get('ignore', False):
-        files['/etc/systemd/network/{}-iface-{}.network'.format(weight, interface)] = {
-            'source': template,
-            'content_type': 'mako',
-            'context': {
-                'interface': interface,
-                'config': config,
-            },
-            'needed_by': {
-                'svc_systemd:systemd-networkd',
-            },
-            'triggers': {
-                'svc_systemd:systemd-networkd:restart',
-            },
-        }
-
-for bond, config in node.metadata.get('systemd-networkd/bonds', {}).items():
-    files['/etc/systemd/network/20-bond-{}.netdev'.format(bond)] = {
-        'source': 'template-bond.netdev',
-        'content_type': 'mako',
-        'context': {
-            'bond': bond,
-            'mode': config.get('mode', '802.3ad'),
-            'prio': config.get('priority', '32768'),
-        },
-        'needed_by': {
-            'svc_systemd:systemd-networkd',
-        },
-        'triggers': {
-            'svc_systemd:systemd-networkd:restart',
-        },
-    }
-    files['/etc/systemd/network/21-bond-{}.network'.format(bond)] = {
-        'source': 'template-bond.network',
-        'content_type': 'mako',
-        'context': {
-            'bond': bond,
-            'match': config['match'],
-        },
-        'needed_by': {
-            'svc_systemd:systemd-networkd',
-        },
-        'triggers': {
-            'svc_systemd:systemd-networkd:restart',
-        },
-    }
-
-for brname, config in node.metadata.get('systemd-networkd/bridges', {}).items():
-    files['/etc/systemd/network/30-bridge-{}.netdev'.format(brname)] = {
-        'source': 'template-bridge.netdev',
-        'content_type': 'mako',
-        'context': {
-            'bridge': brname,
-        },
-        'needed_by': {
-            'svc_systemd:systemd-networkd',
-        },
-        'triggers': {
-            'svc_systemd:systemd-networkd:restart',
-        },
-    }
-    files['/etc/systemd/network/31-bridge-{}.network'.format(brname)] = {
-        'source': 'template-bridge.network',
-        'content_type': 'mako',
-        'context': {
-            'bridge': brname,
-            'match': config['match'],
-        },
-        'needed_by': {
-            'svc_systemd:systemd-networkd',
-        },
-        'triggers': {
-            'svc_systemd:systemd-networkd:restart',
-        },
-    }
 
 svc_systemd = {
     'systemd-networkd': {},

bundles/systemd-networkd/metadata.py

@@ -6,24 +6,8 @@ defaults = {
             },
         },
     },
+    'systemd-networkd': {
+        'netdevs': {},
+        'networks': {},
+    },
 }
-
-
-@metadata_reactor.provides(
-    'interfaces',
-)
-def add_vlan_infos_to_interface(metadata):
-    interfaces = {}
-
-    for iface in metadata.get('interfaces', {}):
-        if not '.' in iface:
-            continue
-
-        interface,vlan = iface.split('.')
-
-        interfaces.setdefault(interface, {}).setdefault('vlans', set())
-        interfaces[interface]['vlans'].add(vlan)
-
-    return {
-        'interfaces': interfaces,
-    }

bundles/systemd/items.py

@@ -12,13 +12,7 @@ actions = {
     }, 
 }
 
-for name, service in node.metadata.get('systemd', {}).get('services', {}).items():
-    # use set() in metadata
-    for enumerator in [
-        'preceded_by', 'needs', 'needed_by', 'triggers', 'triggered_by'
-    ]:
-        assert isinstance(service.get(enumerator, set()), set)
-
+for name, service in node.metadata.get('systemd/services').items():
     # dont call a service 'service' explicitly
     if name.endswith('.service'):
         raise Exception(name)
@@ -34,19 +28,17 @@ for name, service in node.metadata.get('systemd', {}).get('services', {}).items(
     # create unit file
     unit_path = f'/etc/systemd/system/{name}.service'
     files[unit_path] = {
-        'source':       'unitfile',
-        'content_type': 'mako',
-        'context':      {
-            'data': content_data,
-        },
-        'triggers':     [
+        'content': repo.libs.systemd.generate_unitfile(content_data),
+        'triggers': [
             'action:systemd-reload',
             f'svc_systemd:{name}:restart',
         ],
     }
 
     # service depends on unit file
-    service.setdefault('needs', set()).add(f'file:{unit_path}')
+    service\
+        .setdefault('needs', [])\
+        .append(f'file:{unit_path}')
 
     # service
     svc_systemd[name] = service

bundles/systemd/metadata.py

@@ -0,0 +1,5 @@
+defaults = {
+    'systemd': {
+        'services': {},
+    }
+}

bundles/users/items.py

@@ -1,35 +1,28 @@
-from os.path import join, exists
+for group, config in node.metadata.get('groups', {}).items():
+    groups[group] = config
 
-for group, attrs in node.metadata.get('groups', {}).items():
-    groups[group] = attrs
-
-for username, attrs in node.metadata['users'].items():
-    home = attrs.get('home', '/home/{}'.format(username))
-
-    user = users.setdefault(username, {})
-
-    user['home'] = home
-    user['shell'] = attrs.get('shell', '/bin/bash')
-
-    if 'password' in attrs:
-        user['password'] = attrs['password']
-    else:
-        user['password_hash'] = 'x' if node.use_shadow_passwords else '*'
-
-    if 'groups' in attrs:
-        user['groups'] = attrs['groups']
-
-    directories[home] = {
-        'owner': username,
-        'mode': attrs.get('home-mode', '0700'),
+for name, config in node.metadata.get('users').items():
+    directories[config['home']] = {
+        'owner': name,
+        'mode': '700',
     }
 
-    if 'ssh_pubkey' in attrs:
-        files[home + '/.ssh/authorized_keys'] = {
-            'content': '\n'.join(sorted(attrs['ssh_pubkey'])) + '\n',
-            'owner': username,
-            'mode': '0600',
-        }
+    files[f"{config['home']}/.ssh/id_{config['keytype']}"] = {
+        'content': config['privkey'] + '\n',
+        'owner': name,
+        'mode': '0600',
+    }
+    files[f"{config['home']}/.ssh/id_{config['keytype']}.pub"] = {
+        'content': config['pubkey'] + '\n',
+        'owner': name,
+        'mode': '0600',
+    }
+    files[config['home'] + '/.ssh/authorized_keys'] = {
+        'content': '\n'.join(sorted(config['authorized_keys'])) + '\n',
+        'owner': name,
+        'mode': '0600',
+    }
 
-    elif not attrs.get('do_not_remove_authorized_keys_from_home', False):
-        files[home + '/.ssh/authorized_keys'] = {'delete': True}
+    users[name] = config
+    for option in ['authorized_keys', 'privkey', 'pubkey', 'keytype']:
+        users[name].pop(option, None)

bundles/users/metadata.py

@@ -1,9 +1,42 @@
-# defaults = {
-#     'users': {
-#         'root': {
-#             'home': '/root',
-#             'shell': '/bin/bash',
-#             'password': repo.vault.human_password_for('root on {}'.format(node.name)),
-#         },
-#     },
-# }
+from base64 import b64decode
+
+defaults = {
+    'users': {
+        'root': {
+            'home': '/root',
+        },
+    },
+}
+
+
+@metadata_reactor.provides(
+    'users',
+)
+def user(metadata):
+    users = {}
+
+    for name, config in metadata.get('users').items():
+        users[name] = {
+            'authorized_keys': [],
+        }
+
+        if not 'full_name' in config:
+            users[name]['full_name'] = name
+
+        if not 'home' in config:
+            users[name]['home'] = f'/home/{name}'
+
+        if not 'shell' in config:
+            users[name]['shell'] = '/bin/bash'
+            
+        if not 'privkey' in users[name] and not 'pubkey' in users[name]:
+            privkey, pubkey = repo.libs.ssh.generate_ed25519_key_pair(
+                b64decode(str(repo.vault.random_bytes_as_base64_for(f"{name}@{metadata.get('id')}", length=32)))
+            )
+            users[name]['keytype'] = 'ed25519'
+            users[name]['privkey'] = privkey
+            users[name]['pubkey'] = pubkey + f' {name}@{node.name}'
+    
+    return {
+        'users': users,
+    }

bundles/wireguard/files/wg0.netdev

@@ -1,25 +0,0 @@
-[NetDev]
-Name=wg0
-Kind=wireguard
-Description=WireGuard server
-
-[WireGuard]
-PrivateKey=${privatekey}
-ListenPort=51820
-
-% for peer, config in sorted(peers.items()):
-# Peer ${peer}
-[WireGuardPeer]
-PublicKey=${config['pubkey']}
-%   if len(peers) == 1: # FIXME
-AllowedIPs=${network}
-%   else:
-AllowedIPs=${','.join(sorted(config['ips']))}
-%   endif
-PresharedKey=${config['psk']}
-%   if 'endpoint' in config:
-Endpoint=${config['endpoint']}
-%   endif
-PersistentKeepalive=30
-
-% endfor

bundles/wireguard/items.py

@@ -1,21 +1,3 @@
 from ipaddress import ip_network
 
 repo.libs.tools.require_bundle(node, 'systemd-networkd')
-
-network = ip_network(node.metadata['wireguard']['my_ip'], strict=False)
-
-files = {
-    '/etc/systemd/network/wg0.netdev': {
-        'content_type': 'mako',
-        'context': {
-            'network': f'{network.network_address}/{network.prefixlen}',
-            **node.metadata['wireguard'],
-        },
-        'needs': {
-            'pkg_apt:wireguard',
-        },
-        'triggers': {
-            'svc_systemd:systemd-networkd:restart',
-        },
-    },
-}

bundles/wireguard/metadata.py

@@ -1,4 +1,4 @@
-from ipaddress import ip_network
+from ipaddress import ip_network, ip_interface
 
 from bundlewrap.exceptions import NoSuchNode
 from bundlewrap.metadata import atomic
@@ -7,36 +7,100 @@ from bundlewrap.metadata import atomic
 defaults = {
     'apt': {
         'packages': {
-            'wireguard': {},
+            'linux-headers-amd64': {},
+            'wireguard': {
+                'backports': True,
+                'needs': [
+                    'pkg_apt:linux-headers-amd64',
+                ],
+                'triggers': [
+                    'svc_systemd:systemd-networkd:restart',
+                ],
+            },
         },
     },
     'wireguard': {
-        'privatekey': repo.libs.keys.gen_privkey(repo, f'{node.name} wireguard privatekey'),
+        'privatekey': repo.vault.random_bytes_as_base64_for(f'{node.name} wireguard privatekey'),
     },
 }
 
 
 @metadata_reactor.provides(
-    'wireguard/peers',
+    'systemd-networkd/networks',
 )
-def peer_psks(metadata):
-    peers = {}
+def systemd_networkd_networks(metadata):
+    network = {
+        'Match': {
+            'Name': 'wg0',
+        },
+        'Address': {
+            'Address': metadata.get('wireguard/my_ip'),
+        },
+        'Route': {
+            'Destination': str(ip_interface(metadata.get('wireguard/my_ip')).network),
+            'GatewayOnlink': 'yes',
+        },
+        'Network': {
+            'DHCP': 'no',
+            'IPForward': 'yes',
+            'IPv6AcceptRA': 'no',
+        },
+    }
 
-    for peer_name in metadata.get('wireguard/peers', {}):
-        peers[peer_name] = {}
-
-        if node.name < peer_name:
-            peers[peer_name] = {
-                'psk': repo.vault.random_bytes_as_base64_for(f'{node.name} wireguard {peer_name}'),
-            }
-        else:
-            peers[peer_name] = {
-                'psk': repo.vault.random_bytes_as_base64_for(f'{peer_name} wireguard {node.name}'),
-            }
+    for peer, config in metadata.get('wireguard/peers').items():
+        for route in config.get('route', []):
+            network.update({
+                f'Route#{peer}_{route}': {
+                    'Destination': route,
+                    'Gateway': str(ip_interface(repo.get_node(peer).metadata.get(f'wireguard/my_ip')).ip),
+                    'GatewayOnlink': 'yes',
+                }
+            })
 
     return {
-        'wireguard': {
-            'peers': peers,
+        'systemd-networkd': {
+            'networks': {
+                'wireguard': network,
+            },
+        },
+    }
+
+
+@metadata_reactor.provides(
+    'systemd-networkd/netdevs',
+)
+def systemd_networkd_netdevs(metadata):
+    netdev = {
+        'NetDev': {
+            'Name': 'wg0',
+            'Kind': 'wireguard',
+            'Description': 'WireGuard server',
+        },
+        'WireGuard': {
+            'PrivateKey': metadata.get('wireguard/privatekey'),
+            'ListenPort': 51820,
+        },
+    }
+    
+    for peer, config in metadata.get('wireguard/peers').items():
+        netdev.update({
+            f'WireGuardPeer#{peer}': {
+                'Endpoint': config['endpoint'],
+                'PublicKey': config['pubkey'],
+                'PresharedKey': config['psk'],
+                'AllowedIPs': ', '.join([
+                    str(ip_interface(repo.get_node(peer).metadata.get(f'wireguard/my_ip')).ip),
+                    *config.get('route', []),
+                ]), # FIXME
+                'PersistentKeepalive': 30,
+            }
+        })
+    
+    return {
+        'systemd-networkd': {
+            'netdevs': {
+                'wireguard': netdev,
+            },
         },
     }
 
@@ -44,21 +108,24 @@ def peer_psks(metadata):
 @metadata_reactor.provides(
     'wireguard/peers',
 )
-def peer_pubkeys(metadata):
+def peer_keys(metadata):
     peers = {}
 
     for peer_name in metadata.get('wireguard/peers', {}):
-        try:
-            rnode = repo.get_node(peer_name)
-        except NoSuchNode:
-            continue
+        peer_node = repo.get_node(peer_name)
+        
+        first, second = sorted([node.name, peer_name])
+        psk = repo.vault.random_bytes_as_base64_for(f'{first} wireguard {second}')
+        
+        pubkey = repo.libs.keys.get_pubkey_from_privkey(
+            f'{peer_name} wireguard pubkey',
+            peer_node.metadata.get('wireguard/privatekey'),
+        )
         
         peers[peer_name] = {
-            'pubkey': repo.libs.keys.get_pubkey_from_privkey(
-                repo,
-                f'{rnode.name} wireguard pubkey',
-                rnode.metadata.get('wireguard/privatekey'),
-            ),
+            'psk': psk,
+            'pubkey': pubkey,
+            'endpoint': f'{peer_node.hostname}:51820',
         }
 
     return {
@@ -66,75 +133,3 @@ def peer_pubkeys(metadata):
             'peers': peers,
         },
     }
-
-
-@metadata_reactor.provides(
-    'wireguard/peers',
-)
-def peer_ips_and_endpoints(metadata):
-    peers = {}
-
-    for peer_name in metadata.get('wireguard/peers', {}):
-        try:
-            rnode = repo.get_node(peer_name)
-        except NoSuchNode:
-            continue
-
-        ips = rnode.metadata.get('wireguard/subnets', set())
-        ips.add(rnode.metadata.get('wireguard/my_ip').split('/')[0])
-        ips = repo.libs.tools.remove_more_specific_subnets(ips)
-
-        peers[rnode.name] = {
-            'endpoint': '{}:51820'.format(rnode.metadata.get('wireguard/external_hostname', rnode.hostname)),
-            'ips': ips,
-        }
-
-    return {
-        'wireguard': {
-            'peers': peers,
-        },
-    }
-
-
-@metadata_reactor.provides(
-    'interfaces/wg0/ips',
-)
-def interface_ips(metadata):
-    return {
-        'interfaces': {
-            'wg0': {
-                'ips': {
-                    metadata.get('wireguard/my_ip'),
-                },
-            },
-        },
-    }
-
-
-@metadata_reactor.provides(
-    'interfaces/wg0/routes',
-)
-def routes(metadata):
-    network = ip_network(metadata.get('wireguard/my_ip'), strict=False)
-    ips = {
-        f'{network.network_address}/{network.prefixlen}',
-    }
-    routes = {}
-
-    for _, peer_config in metadata.get('wireguard/peers', {}).items():
-        for ip in peer_config['ips']:
-            ips.add(ip)
-
-    if '0.0.0.0/0' in ips:
-        ips.remove('0.0.0.0/0')
-
-    for ip in repo.libs.tools.remove_more_specific_subnets(ips):
-        routes[ip] = {}
-
-    return {
-        'interfaces': {
-            'wg0': {
-                'routes': routes,
-            },
-        },
-    }

bundles/zfs/items.py

@@ -26,15 +26,17 @@ svc_systemd = {
     },
 }
 
-zfs_datasets = node.metadata.get('zfs/datasets')
+for name, config in node.metadata.get('zfs/datasets', {}).items():
+    zfs_datasets[name] = config
+    zfs_datasets[name].pop('backup', None)
 
-for name, attrs in node.metadata.get('zfs/pools', {}).items():
-    zfs_pools[name] = attrs
+for name, config in node.metadata.get('zfs/pools', {}).items():
+    zfs_pools[name] = config
 
-    # actions[f'pool_{name}_enable_trim'] = {
-    #    'command': f'zpool set autotrim=on {name}',
-    #    'unless':  f'zpool get autotrim -H -o value {name} | grep -q on',
-    #    'needs':   [
-    #        f'zfs_pool:{name}'
-    #    ]
-    # }
+    actions[f'pool_{name}_enable_trim'] = {
+       'command': f'zpool set autotrim=on {name}',
+       'unless':  f'zpool get autotrim -H -o value {name} | grep -q on',
+       'needs':   [
+           f'zfs_pool:{name}'
+       ]
+    }

bundles/zfs/metadata.py

@@ -8,19 +8,28 @@ defaults = {
                     'pkg_apt:zfs-dkms',
                 },
             },
+            'parted':{
+                'needed_by': {
+                    'pkg_apt:zfs-zed',
+                    'pkg_apt:zfsutils-linux',
+                },
+            },
             'zfs-dkms': {
+                'backports': True,
                 'needed_by': {
                     'pkg_apt:zfs-zed',
                     'pkg_apt:zfsutils-linux',
                 },
             },
             'zfs-zed': {
+                'backports': True,
                 'needed_by': {
                     'zfs_dataset:',
                     'zfs_pool:',
                 },
             },
             'zfsutils-linux': {
+                'backports': True,
                 'needed_by': {
                     'pkg_apt:zfs-zed',
                     'zfs_dataset:',
@@ -48,3 +57,17 @@ def dataset_defaults(metadata):
             },
         },
     }
+
+
+@metadata_reactor.provides(
+    'backup/paths'
+)
+def backup(metadata):
+    return {
+        'backup': {
+            'paths': [
+                options['mountpoint'] for options in metadata.get('zfs/datasets').values()
+                    if options.get('backup', True)
+            ],
+        },
+    }

data/apt/keys/repos.influxdata.com.asc

@@ -0,0 +1,52 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mQINBFYJmwQBEADCw7mob8Vzk+DmkYyiv0dTU/xgoSlp4SQwrTzat8MB8jxmx60l
+QjmhqEyuB8ho4zzZF9KV+gJWrG6Rj4t69JMTJWM7jFz+0B1PC7kJfNM+VcBmkTnj
+fP+KJjqz50ETnsF0kQTG++UJeRYjG1dDK0JQNQJAM6NQpIWJI339lcDf15vzrMnb
+OgIlNxV6j1ZZqkle4fvScF1NQxYScRiL+sRgVx92SI4SyD/xZnVGD/szB+4OCzah
++0Q/MnNGV6TtN0RiCDZjIUYiHoeT9iQXEONKf7T62T4zUafO734HyqGvht93MLVU
+GQAeuyx0ikGsULfOsJfBmb3XJS9u+16v7oPFt5WIbeyyNuhUu0ocK/PKt5sPYR4u
+ouPq6Ls3RY3BGCH9DpokcYsdalo51NMrMdnYwdkeq9MEpsEKrKIN5ke7fk4weamJ
+BiLI/bTcfM7Fy5r4ghdI9Ksw/ULXLm4GNabkIOSfT7UjTzcBDOvWfKRBLX4qvsx4
+YzA5kR+nX85u6I7W10aSqBiaLqk6vCj0QmBmCjlSeYqNQqSzH/6OoL6FZ7lP6AiG
+F2NyGveJKjugoXlreLEhOYp20F81PNwlRBCAlMC2Q9mpcFu0dtAriVoG4gVDdYn5
+t+BiGfD2rJlCinYLgYBDpTPcdRT3VKHWqL9fcC4HKmic0mwWg9homx550wARAQAB
+tDFJbmZsdXhEQiBQYWNrYWdpbmcgU2VydmljZSA8c3VwcG9ydEBpbmZsdXhkYi5j
+b20+iQI3BBMBCgAhBQJWCZsEAhsDBQsJCAcDBRUKCQgLBRYDAgEAAh4BAheAAAoJ
+EGhKFM8lguDF9XEQAK9rREnZt6ujh7GXfeNki35bkn39q8GYh0mouShFbFY9o0i3
+UJVChsxokJSRPgFh9GOhOPTupl3rzfdpD+IlWI2Myt6han2HOjZKNZ4RGNrYJ5UR
+uxt4dKMWlMbpkzL56bhHlx97RoXKv2d2zRQfw9nyZb6t3lw2k2kKXsMxjGa0agM+
+2SropwYOXdtkz8UWaGd3LYxwEvW3AuhI8EEEHdLetQaYe9sANDvUEofgFbdsuICH
+9QLmbYavk7wyGTPBKfPBbeyTxwW2rMUnFCNccMKLm1i5NpZYineBtQbX2cfx9Xsk
+1JLOzEBmNal53H2ob0kjev6ufzOD3s8hLu4KMCivbIz4YT3fZyeExn0/0lUtsQ56
+5fCxE983+ygDzKsCnfdXqm3GgjaI90OkNr1y4gWbcd5hicVDv5fD3TD9f0GbpDVw
+yDz8YmvNzxMILt5Glisr6aH7gLG/u8jxy0D8YcBiyv5kfY4vMI2yXHpGg1cn/sVu
+ZB01sU09VVIM2BznnimyAayI430wquxkZCyMx//BqFM1qetIgk1wDZTlFd0n6qtA
+fDmXAC4s5pM5rfM5V57WmPaIqnRIaESJ35tFUFlCHfkfl/N/ribGVDg1z2KDW08r
+96oEiIIiV4GfXl+NprJqpNS3Cn+aCXtd7/TsDScDEgs4sMaR29Lsf26cuWk8uQIN
+BFYJmwQBEADDPi3fmwn6iwkiDcH2E2V31cHlBw9OdJfxKVUdyAQEhTtqmG9P8XFZ
+ERRQF155XLQPLvRlUlq7vEYSROn5J6BAnsjdjsH9LmFMOEV8CIRCRIDePG/Mez2d
+nIK5yiU6GkS3IFaQg2T9/tOBKxm0ZJPfqTXbT4jFSfvYJ3oUqc+AyYxtb8gj1GRk
+X283/86/bA3C98u7re1vPtiDRyM8r0+lhEc59Yx/EAOL+X2gZyTgyUoH+LLuOWQK
+s1egI8y80R8NZfM1nMiQk2ywMsTFwQjSVimScvzqv5Nt8k8CvHUQ3a6R+6doXGNX
+5RnUqn9Qvmh0JY5sNgFsoaGbuk2PJrVaGBRnfnjaDqAlZpDhwkWhcCcguNhRbRHp
+N7/a0pQr70bAG9VikzLyGC17EU0sxney/hyNHkr4Uyy2OXHpuJvRjVKy/BwZ3fxA
+AYX2oZIOxQB3/OulzO/DppaCVhRtp1bt+Z5f+fpisiVb5DvZcMdeyAoQ4+oOr7v3
+EasIs2XYcQ+kOE3Y2kdlHWBeuXzxgWgJZ1OOpwGMjR3Uy6IwhuSWtreJBA4er+Df
+vgSPwKBsRLNLbPe3ftjArnC5GfMiGgikVdAUdN4OkEqvUbkRoAVGKTOMLUKm+ZkG
+OskJOVYS+JAina0qkYEFF7haycMjf9olhqLmTIC+6X7Ox9R2plaOhQARAQABiQIf
+BBgBCgAJBQJWCZsEAhsMAAoJEGhKFM8lguDF8ZIP/1q9Sdz8oMvf9AJXZ7AYxm77
+V+kJzJqi62nZLWJnrFXDZJpU+LkYlb3fstsZ1rvBhnrEPSmFxoj72CP0RtcyX7wJ
+dA7K1Fl9LpJi5H8300cC7UyG94MUYbrXijbLTbnFTfNr1tGx4a1T/7Yyxx/wZGrT
+H/X8cvNybkl33SxDdlQQ9kx3lFOwC41e3TkGsUWxn3TCfvDh8VdA6Py6JeSPFGOb
+MEO2/q7oUgvjfV+ivN5ayZi9bWgeqm1sgtmTHHQ4RqwwKrAb5ynXpn1b9QrkevgT
+b91uzMA22Prl4DuzKiaMYDcZOQ3vtf0eFBP0GOSSgUKS4bQ3dGgi1JmQ7VuAM4uj
++Ug5TnGoLwclTwLksc7v89C5MMPgm2vVXvCUDzyzQA7bIHFeX+Rziby4nymec4Nr
+eeXYNBJWrEp8XR7UNWmEgroXRoN1x9/6esh5pnoUXGAIWuKzSLQM70/wWxS67+v2
+aC1GNb+pXXAzYeIIiyLWaZwCSr8sWMvshFT9REk2+lnb6sAeJswQtfTUWI00mVqZ
+dvI3Wys2h0IyIejuwetTUvGhr9VgpqiLLfGzGlt/y2sg27wdHzSJbMh0VrVAK26/
+BlvEwWDCFT0ZJUMG9Lvre25DD0ycbougLsRYjzmGb/3k3UktS3XTCxyBa/k3TPw3
+vqIHrEqk446nGPDqJPS5
+=9iF7
+-----END PGP PUBLIC KEY BLOCK-----