dist-upgrade -> full-upgrade

Reviewed-on: #18

.envrc

@@ -1,15 +1,7 @@
 #!/usr/bin/env bash
 
-python3 -m venv .venv
-source ./.venv/bin/activate
-PATH_add .venv/bin
 PATH_add bin
-python3 -m pip install --upgrade pip
 
-rm -rf .cache/bw/git_deploy
-export BW_GIT_DEPLOY_CACHE=.cache/bw/git_deploy
-mkdir -p "$BW_GIT_DEPLOY_CACHE"
-export EXPERIMENTAL_UPLOAD_VIA_CAT=1
-export BW_ITEM_WORKERS=64
-export BW_NODE_WORKERS=16
-unset PS1
+source_env ~/.local/share/direnv/pyenv
+source_env ~/.local/share/direnv/venv
+source_env ~/.local/share/direnv/bundlewrap

README.md

@@ -37,3 +37,12 @@ fi
 telegraf: execd for daemons
 
 TEST
+
+# git signing
+
+git config --global gpg.format ssh
+git config --global commit.gpgsign true
+
+git config user.name CroneKorkN
+git config user.email i@ckn.li
+git config user.signingkey "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILMVroYmswD4tLk6iH+2tvQiyaMe42yfONDsPDIdFv6I"

bin/rcon

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+from sys import argv
+from os.path import realpath, dirname
+from shlex import quote
+from bundlewrap.repo import Repository
+
+repo = Repository(dirname(dirname(realpath(__file__))))
+
+if len(argv) == 1:
+    for node in repo.nodes:
+        for name in node.metadata.get('left4dead2/servers', {}):
+            print(name)
+    exit(0)
+
+server = argv[1]
+command = argv[2]
+
+remote_code = """
+from rcon.source import Client
+
+with Client('127.0.0.1', {port}, passwd='''{password}''') as client:
+    response = client.run('''{command}''')
+
+print(response)
+"""
+
+for node in repo.nodes:
+    for name, conf in node.metadata.get('left4dead2/servers', {}).items():
+        if name == server:
+            response = node.run('python3 -c ' + quote(remote_code.format(port=conf['port'], password=conf['rcon_password'], command=command)))
+            print(response.stdout.decode())

bin/upgrade_and_restart_all

@@ -10,7 +10,6 @@ nodes = [
         for node in sorted(repo.nodes_in_group('debian'))
         if not node.dummy
 ]
-reboot_nodes = []
 
 print('updating nodes:', sorted(node.name for node in nodes))
 
@@ -24,14 +23,13 @@ for node in nodes:
     print(node.run('DEBIAN_FRONTEND=noninteractive apt update').stdout.decode())
     print(node.run('DEBIAN_FRONTEND=noninteractive apt list --upgradable').stdout.decode())
     if int(node.run('DEBIAN_FRONTEND=noninteractive apt list --upgradable 2> /dev/null | grep upgradable | wc -l').stdout.decode()):
-        print(node.run('DEBIAN_FRONTEND=noninteractive apt -y dist-upgrade').stdout.decode())
-        reboot_nodes.append(node)
+        print(node.run('DEBIAN_FRONTEND=noninteractive apt -qy full-upgrade').stdout.decode())
 
 # REBOOT IN ORDER
 
 wireguard_servers = [
     node
-        for node in reboot_nodes
+        for node in nodes
         if node.has_bundle('wireguard')
         and (
             ip_interface(node.metadata.get('wireguard/my_ip')).network.prefixlen <
@@ -41,7 +39,7 @@ wireguard_servers = [
 
 wireguard_s2s = [
     node
-        for node in reboot_nodes
+        for node in nodes
         if node.has_bundle('wireguard')
         and (
             ip_interface(node.metadata.get('wireguard/my_ip')).network.prefixlen ==
@@ -51,7 +49,7 @@ wireguard_s2s = [
 
 everything_else = [
     node
-        for node in reboot_nodes
+        for node in nodes
         if not node.has_bundle('wireguard')
 ]
 
@@ -62,8 +60,11 @@ for node in [
     *wireguard_s2s,
     *wireguard_servers,
 ]:
-    print('rebooting', node.name)
     try:
-        print(node.run('systemctl reboot').stdout.decode())
+        if node.run('test -e /var/run/reboot-required', may_fail=True).return_code == 0:
+            print('rebooting', node.name)
+            print(node.run('systemctl reboot').stdout.decode())
+        else:
+            print('not rebooting', node.name)
     except Exception as e:
         print(e)

bin/wireguard_client_config

@@ -5,9 +5,17 @@ from os.path import realpath, dirname
 from sys import argv
 from ipaddress import ip_network, ip_interface
 
-repo = Repository(dirname(dirname(realpath(__file__))))
+if len(argv) != 3:
+    print(f'usage: {argv[0]} <node> <client>')
+    exit(1)
 
+repo = Repository(dirname(dirname(realpath(__file__))))
 server_node = repo.get_node(argv[1])
+
+if argv[2] not in server_node.metadata.get('wireguard/clients'):
+    print(f'client {argv[2]} not found in: {server_node.metadata.get("wireguard/clients").keys()}')
+    exit(1)
+
 data = server_node.metadata.get(f'wireguard/clients/{argv[2]}')
 
 vpn_network = ip_interface(server_node.metadata.get('wireguard/my_ip')).network
@@ -20,9 +28,7 @@ for peer in server_node.metadata.get('wireguard/s2s').values():
         if not ip_network(network).subnet_of(vpn_network):
             allowed_ips.append(ip_network(network))
 
-conf = \
-f'''>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
-
+conf = f'''
 [Interface]
 PrivateKey = {repo.libs.wireguard.privkey(data['peer_id'])}
 ListenPort = 51820
@@ -35,11 +41,12 @@ PresharedKey = {repo.libs.wireguard.psk(data['peer_id'], server_node.metadata.ge
 AllowedIPs = {', '.join(str(client_route) for client_route in sorted(allowed_ips))}
 Endpoint = {ip_interface(server_node.metadata.get('network/external/ipv4')).ip}:51820
 PersistentKeepalive = 10
+'''
 
-<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'''
-
+print('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
 print(conf)
+print('<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<')
 
-if input("print qrcode? [yN]: ").upper() == 'Y':
+if input("print qrcode? [Yn]: ").upper() in ['', 'Y']:
     import pyqrcode
     print(pyqrcode.create(conf).terminal(quiet_zone=1))

bundles/apt/README.md

@@ -1,3 +1,6 @@
+# https://manpages.debian.org/latest/apt/sources.list.5.de.html
+# https://repolib.readthedocs.io/en/latest/deb822-format.html
+
 ```python
 {
     'apt': {
@@ -5,8 +8,29 @@
             'apt-transport-https': {},
         },
         'sources': {
-            # place key under data/apt/keys/packages.cloud.google.com.{asc|gpg}
-            'deb https://packages.cloud.google.com/apt cloud-sdk main',
+            'debian': {
+                'types': { # optional, defaults to `{'deb'}``
+                    'deb',
+                    'deb-src',
+                },
+                'urls': {
+                    'https://deb.debian.org/debian',
+                },
+                'suites': { # at least one
+                    '{codename}',
+                    '{codename}-updates',
+                    '{codename}-backports',
+                },
+                'components': { # optional
+                    'main',
+                    'contrib',
+                    'non-frese',
+                },
+                # key:
+                # - optional, defaults to source name (`debian` in this example)
+                # - place key under data/apt/keys/debian-12.{asc|gpg}
+                'key': 'debian-{version}',
+            },
         },
     },
 }

bundles/apt/files/20auto-upgrades

@@ -1,2 +0,0 @@
-APT::Periodic::Update-Package-Lists "1";
-APT::Periodic::Unattended-Upgrade "1";

bundles/apt/files/50unattended-upgrades

@@ -1,3 +0,0 @@
-Unattended-Upgrade::Origins-Pattern {
-  "origin=*";
-};

bundles/apt/items.py

@@ -1,33 +1,64 @@
-from os.path import join
-from urllib.parse import urlparse
-from glob import glob
+# TODO pin repo: https://superuser.com/a/1595920
+
 from os.path import join, basename
 
 directories = {
-    '/etc/apt/sources.list.d': {
+    '/etc/apt': {
         'purge': True,
         'triggers': {
             'action:apt_update',
         },
     },
-    '/etc/apt/trusted.gpg.d': {
+    '/etc/apt/apt.conf.d': {
+        # existance is expected
         'purge': True,
         'triggers': {
             'action:apt_update',
         },
     },
+    '/etc/apt/keyrings': {
+        # https://askubuntu.com/a/1307181
+        'purge': True,
+        'triggers': {
+            'action:apt_update',
+        },
+    },
+    # '/etc/apt/listchanges.conf.d': {
+    #     'purge': True,
+    #     'triggers': {
+    #         'action:apt_update',
+    #     },
+    # },
     '/etc/apt/preferences.d': {
         'purge': True,
         'triggers': {
             'action:apt_update',
         },
     },
+    '/etc/apt/sources.list.d': {
+        'purge': True,
+        'triggers': {
+            'action:apt_update',
+        },
+    },
 }
 
 files = {
-    '/etc/apt/sources.list': {
-        'content': '# managed'
+    '/etc/apt/apt.conf': {
+        'content': repo.libs.apt.render_apt_conf(node.metadata.get('apt/config')),
+        'triggers': {
+            'action:apt_update',
+        },
     },
+    '/etc/apt/sources.list': {
+        'content': '# managed by bundlewrap\n',
+        'triggers': {
+            'action:apt_update',
+        },
+    },
+    # '/etc/apt/listchanges.conf': {
+    #     'content': repo.libs.ini.dumps(node.metadata.get('apt/list_changes')),
+    # },
     '/usr/lib/nagios/plugins/check_apt_upgradable': {
         'mode': '0755',
     },
@@ -44,41 +75,22 @@ actions = {
     },
 }
 
-# group sources by apt server hostname
+# create sources.lists and respective keyfiles
 
-hosts = {}
-
-for source_string in node.metadata.get('apt/sources'):
-    source = repo.libs.apt.AptSource(source_string)
-    hosts\
-        .setdefault(source.url.hostname, list())\
-        .append(source)
-
-# create sources lists and keyfiles
-
-for host, sources in hosts.items():
-    keyfile = basename(glob(join(repo.path, 'data', 'apt', 'keys', f'{host}.*'))[0])
-    destination_path = f'/etc/apt/trusted.gpg.d/{keyfile}'
-
-    for source in sources:
-        source.options['signed-by'] = [destination_path]
-
-    files[f'/etc/apt/sources.list.d/{host}.list'] = {
-        'content': '\n'.join(sorted(set(
-            str(source).format(
-                release=node.metadata.get('os_release'),
-                version=node.os_version[0], # WIP crystal
-            )
-                for source in sources
-        ))),
+for name, config in node.metadata.get('apt/sources').items():
+    # place keyfile
+    keyfile_destination_path = repo.libs.apt.format_variables(node, config['options']['Signed-By'])
+    files[keyfile_destination_path] = {
+        'source': join(repo.path, 'data', 'apt', 'keys', basename(keyfile_destination_path)),
+        'content_type': 'binary',
         'triggers': {
             'action:apt_update',
         },
     }
 
-    files[destination_path] = {
-        'source': join(repo.path, 'data', 'apt', 'keys', keyfile),
-        'content_type': 'binary',
+    # place sources.list
+    files[f'/etc/apt/sources.list.d/{name}.sources'] = {
+        'content': repo.libs.apt.render_source(node, name),
         'triggers': {
             'action:apt_update',
         },
@@ -93,7 +105,7 @@ for package, options in node.metadata.get('apt/packages', {}).items():
         files[f'/etc/apt/preferences.d/{package}'] = {
             'content': '\n'.join([
                 f"Package: {package}",
-                f"Pin: release a={node.metadata.get('os_release')}-backports",
+                f"Pin: release a={node.metadata.get('os_codename')}-backports",
                 f"Pin-Priority: 900",
             ]),
             'needed_by': [
@@ -110,8 +122,6 @@ for package, options in node.metadata.get('apt/packages', {}).items():
 # apt-daily.timer: performs apt update
 # apt-daily-upgrade.timer: performs apt upgrade
 
-files['/etc/apt/apt.conf.d/20auto-upgrades'] = {}
-files['/etc/apt/apt.conf.d/50unattended-upgrades'] = {}
 svc_systemd['unattended-upgrades.service'] = {
     'needs': [
         'pkg_apt:unattended-upgrades',

bundles/apt/metadata.py

@@ -1,9 +1,55 @@
 defaults = {
     'apt': {
         'packages': {
-            'unattended-upgrades': {},
+            'apt-listchanges': {
+                'installed': False,
+            },
         },
-        'sources': set(),
+        'config': {
+            'DPkg': {
+                'Pre-Install-Pkgs': {
+                    '/usr/sbin/dpkg-preconfigure --apt || true',
+                },
+                'Post-Invoke': {
+                    # keep package cache empty
+                    '/bin/rm -f /var/cache/apt/archives/*.deb || true',
+                },
+                'Options': {
+                    # https://unix.stackexchange.com/a/642541/357916
+                    '--force-confold',
+                    '--force-confdef',
+                },
+            },
+            'APT': {
+                'NeverAutoRemove': {
+                    '^firmware-linux.*',
+                    '^linux-firmware$',
+                    '^linux-image-[a-z0-9]*$',
+                    '^linux-image-[a-z0-9]*-[a-z0-9]*$',
+                },
+                'VersionedKernelPackages': {
+                    # kernels
+                    'linux-.*',
+                    'kfreebsd-.*',
+                    'gnumach-.*',
+                    # (out-of-tree) modules
+                    '.*-modules',
+                    '.*-kernel',
+                },
+                'Never-MarkAuto-Sections': {
+                    'metapackages',
+                    'tasks',
+                },
+                'Move-Autobit-Sections': {
+                    'oldlibs',
+                },
+                'Update': {
+                    # https://unix.stackexchange.com/a/653377/357916
+                    'Error-Mode': 'any',
+                },
+            },
+        },
+        'sources': {},
     },
     'monitoring': {
         'services': {
@@ -23,3 +69,109 @@ defaults = {
         },
     },
 }
+
+
+@metadata_reactor.provides(
+    'apt/sources',
+)
+def key(metadata):
+    return {
+        'apt': {
+            'sources': {
+                source_name: {
+                    'key': source_name,
+                }
+                    for source_name, source_config in metadata.get('apt/sources').items()
+                    if 'key' not in source_config
+            },
+        },
+    }
+
+
+@metadata_reactor.provides(
+    'apt/sources',
+)
+def signed_by(metadata):
+    return {
+        'apt': {
+            'sources': {
+                source_name: {
+                    'options': {
+                        'Signed-By': '/etc/apt/keyrings/' + metadata.get(f'apt/sources/{source_name}/key') + '.' + repo.libs.apt.find_keyfile_extension(node, metadata.get(f'apt/sources/{source_name}/key')),
+                    },
+                }
+                    for source_name in metadata.get('apt/sources')
+            },
+        },
+    }
+
+
+@metadata_reactor.provides(
+    'apt/config',
+    'apt/packages',
+)
+def unattended_upgrades(metadata):
+    return {
+        'apt': {
+            'config': {
+                'APT': {
+                    'Periodic': {
+                        'Update-Package-Lists': '1',
+                        'Unattended-Upgrade': '1',
+                    },
+                },
+                'Unattended-Upgrade': {
+                    'Origins-Pattern': {
+                        "origin=*",
+                    },
+                },
+            },
+            'packages': {
+                'unattended-upgrades': {},
+            },
+        },
+    }
+
+
+# @metadata_reactor.provides(
+#     'apt/config',
+#     'apt/list_changes',
+# )
+# def listchanges(metadata):
+#     return {
+#         'apt': {
+#             'config': {
+#                 'DPkg': {
+#                     'Pre-Install-Pkgs': {
+#                         '/usr/bin/apt-listchanges --apt || test $? -lt 10',
+#                     },
+#                     'Tools': {
+#                         'Options': {
+#                             '/usr/bin/apt-listchanges': {
+#                                 'Version': '2',
+#                                 'InfoFD': '20',
+#                             },
+#                         },
+#                     },
+#                 },
+#                 'Dir': {
+#                     'Etc': {
+#                         'apt-listchanges-main': 'listchanges.conf',
+#                         'apt-listchanges-parts': 'listchanges.conf.d',
+#                     },
+#                 },
+#             },
+#             'list_changes': {
+#                 'apt': {
+#                     'frontend': 'pager',
+#                     'which': 'news',
+#                     'email_address': 'root',
+#                     'email_format': 'text',
+#                     'confirm': 'false',
+#                     'headers': 'false',
+#                     'reverse': 'false',
+#                     'save_seen': '/var/lib/apt/listchanges.db',
+#                 },
+#             },
+#         },
+#     }

bundles/backup-freshness-check/files/check_backup_freshness

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+import json
+from subprocess import check_output
+from datetime import datetime, timedelta
+
+
+now = datetime.now()
+two_days_ago = now - timedelta(days=2)
+
+with open('/etc/backup-freshness-check.json', 'r') as file:
+    config = json.load(file)
+
+local_datasets = check_output(['zfs', 'list', '-H', '-o', 'name']).decode().splitlines()
+errors = set()
+
+for dataset in config['datasets']:
+    if f'tank/{dataset}' not in local_datasets:
+        errors.add(f'dataset "{dataset}" not present at all')
+        continue
+
+    snapshots = [
+        snapshot
+            for snapshot in check_output(['zfs', 'list', '-H', '-o', 'name', '-t', 'snapshot', f'tank/{dataset}', '-s', 'creation']).decode().splitlines()
+            if f"@{config['prefix']}" in snapshot
+    ]
+
+    if not snapshots:
+        errors.add(f'dataset "{dataset}" has no backup snapshots')
+        continue
+
+    newest_backup_snapshot = snapshots[-1]
+    snapshot_datetime = datetime.utcfromtimestamp(
+        int(check_output(['zfs', 'list', '-p', '-H', '-o', 'creation', '-t', 'snapshot', newest_backup_snapshot]).decode())
+    )
+
+    if snapshot_datetime < two_days_ago:
+        days_ago = (now - snapshot_datetime).days
+        errors.add(f'dataset "{dataset}" has not been backed up for {days_ago} days')
+        continue
+
+if errors:
+    for error in errors:
+        print(error)
+    exit(2)
+else:
+    print(f"all {len(config['datasets'])} datasets have fresh backups.")

bundles/backup-freshness-check/items.py

@@ -0,0 +1,15 @@
+from json import dumps
+from bundlewrap.metadata import MetadataJSONEncoder
+
+
+files = {
+    '/etc/backup-freshness-check.json': {
+        'content': dumps({
+            'prefix': node.metadata.get('backup-freshness-check/prefix'),
+            'datasets': node.metadata.get('backup-freshness-check/datasets'),
+        }, indent=4, sort_keys=True, cls=MetadataJSONEncoder),
+    },
+    '/usr/lib/nagios/plugins/check_backup_freshness': {
+        'mode': '0755',
+    },
+}

bundles/backup-freshness-check/metadata.py

@@ -0,0 +1,37 @@
+defaults = {
+    'backup-freshness-check': {
+        'server': node.name,
+        'prefix': 'auto-backup_',
+        'datasets': {},
+    },
+    'monitoring': {
+        'services': {
+            'backup freshness': {
+                'vars.command': '/usr/lib/nagios/plugins/check_backup_freshness',
+                'check_interval': '6h',
+                'vars.sudo': True,
+            },
+        },
+    },
+}
+
+
+@metadata_reactor.provides(
+    'backup-freshness-check/datasets'
+)
+def backup_freshness_check(metadata):
+    return {
+        'backup-freshness-check': {
+            'datasets': {
+                f"{other_node.metadata.get('id')}/{dataset}"
+                    for other_node in repo.nodes
+                    if not other_node.dummy
+                    and other_node.has_bundle('backup')
+                    and other_node.has_bundle('zfs')
+                    and other_node.metadata.get('backup/server') == metadata.get('backup-freshness-check/server')
+                    for dataset, options in other_node.metadata.get('zfs/datasets').items()
+                    if options.get('backup', True)
+                    and not options.get('mountpoint', None) in [None, 'none']
+            },
+        },
+    }

bundles/backup-server/metadata.py

@@ -35,6 +35,7 @@ def zfs(metadata):
 
     for other_node in repo.nodes:
         if (
+            not other_node.dummy and
             other_node.has_bundle('backup') and
             other_node.metadata.get('backup/server') == node.name
         ):

bundles/backup/files/backup_all

@@ -1,11 +1,31 @@
 #!/bin/bash
 
+set -u
+
 # FIXME: inelegant
 % if wol_command:
 ${wol_command}
 % endif
 
+exit=0
+failed_paths=""
+
 for path in $(jq -r '.paths | .[]' < /etc/backup/config.json)
 do
+  echo backing up $path
   /opt/backup/backup_path "$path"
+  # set exit to 1 if any backup fails
+  if [ $? -ne 0 ]
+  then
+    echo ERROR: backing up $path failed >&2
+    exit=5
+    failed_paths="$failed_paths $path"
+  fi
 done
+
+if [ $exit -ne 0 ]
+then
+  echo "ERROR: failed to backup paths: $failed_paths" >&2
+fi
+
+exit $exit

bundles/backup/files/backup_path

@@ -1,11 +1,13 @@
 #!/bin/bash
 
+set -exu
+
 path=$1
 
-if zfs list -H -o mountpoint | grep -q "$path"
+if zfs list -H -o mountpoint | grep -q "^$path$"
 then
   /opt/backup/backup_path_via_zfs "$path"
-elif test -d "$path"
+elif test -e "$path"
 then
   /opt/backup/backup_path_via_rsync "$path"
 else

bundles/backup/files/backup_path_via_rsync

@@ -7,5 +7,14 @@ uuid=$(jq -r .client_uuid < /etc/backup/config.json)
 server=$(jq -r .server_hostname < /etc/backup/config.json)
 ssh="ssh -o ConnectTimeout=5 backup-receiver@$server"
 
-rsync -av --rsync-path="sudo rsync" "$path/" "backup-receiver@$server:/mnt/backups/$uuid$path/"
-$ssh sudo zfs snap "tank/$uuid/fs@auto-backup_$(date +"%Y-%m-%d_%H:%M:%S")" 
+if test -d "$path"
+then
+  postfix="/"
+elif test -f "$path"
+then
+  postfix=""
+else
+  exit 1
+fi
+
+rsync -av --rsync-path="sudo rsync" "$path$postfix" "backup-receiver@$server:/mnt/backups/$uuid$path$postfix"

bundles/backup/files/backup_path_via_zfs

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -exu
+set -eu
 
 path=$1
 uuid=$(jq -r .client_uuid < /etc/backup/config.json)
@@ -46,13 +46,13 @@ if [[ "$?" == "0" ]]
 then
 
   # delete old local bookmarks
-  for destroyable_bookmark in $(zfs list -t bookmark -H -o name "$dataset" | grep "^$dataset#$bookmark_prefix")
+  for destroyable_bookmark in $(zfs list -t bookmark -H -o name "$source_dataset" | grep "^$source_dataset#$bookmark_prefix")
   do
     zfs destroy "$destroyable_bookmark"
   done
 
-  # delete snapshots from bookmarks (except newest, even of not necessary; maybe for resuming tho)
-  for destroyable_snapshot in $($ssh sudo zfs list -t snapshot -H -o name "$dataset" | grep "^$dataset@$bookmark_prefix" | grep -v "$new_bookmark")
+  # delete remote snapshots from bookmarks (except newest, even of not necessary; maybe for resuming tho)
+  for destroyable_snapshot in $($ssh sudo zfs list -t snapshot -H -o name "$target_dataset" | grep "^$target_dataset@$bookmark_prefix" | grep -v "$new_bookmark")
   do
     $ssh sudo zfs destroy "$destroyable_snapshot"
   done

bundles/bind/items.py

@@ -19,7 +19,7 @@ directories[f'/var/lib/bind'] = {
         'svc_systemd:bind9',
     ],
     'triggers': [
-        'svc_systemd:bind9:restart',
+        'svc_systemd:bind9:reload',
     ],
 }
 
@@ -29,7 +29,7 @@ files['/etc/default/bind9'] = {
         'svc_systemd:bind9',
     ],
     'triggers': [
-        'svc_systemd:bind9:restart',
+        'svc_systemd:bind9:reload',
     ],
 }
 
@@ -43,7 +43,7 @@ files['/etc/bind/named.conf'] = {
         'svc_systemd:bind9',
     ],
     'triggers': [
-        'svc_systemd:bind9:restart',
+        'svc_systemd:bind9:reload',
     ],
 }
 
@@ -63,7 +63,7 @@ files['/etc/bind/named.conf.options'] = {
         'svc_systemd:bind9',
     ],
     'triggers': [
-        'svc_systemd:bind9:restart',
+        'svc_systemd:bind9:reload',
     ],
 }
 
@@ -93,7 +93,7 @@ files['/etc/bind/named.conf.local'] = {
         'svc_systemd:bind9',
     ],
     'triggers': [
-        'svc_systemd:bind9:restart',
+        'svc_systemd:bind9:reload',
     ],
 }
 
@@ -106,7 +106,7 @@ for view_name, view_conf in master_node.metadata.get('bind/views').items():
             'svc_systemd:bind9',
         ],
         'triggers': [
-            'svc_systemd:bind9:restart',
+            'svc_systemd:bind9:reload',
         ],
     }
 
@@ -127,7 +127,7 @@ for view_name, view_conf in master_node.metadata.get('bind/views').items():
                 'svc_systemd:bind9',
             ],
             'triggers': [
-                'svc_systemd:bind9:restart',
+                'svc_systemd:bind9:reload',
             ],
         }
 
@@ -139,6 +139,6 @@ actions['named-checkconf'] = {
     'unless': 'named-checkconf -z',
     'needs': [
         'svc_systemd:bind9',
-        'svc_systemd:bind9:restart',
+        'svc_systemd:bind9:reload',
     ]
 }

bundles/build-ci/metadata.py

@@ -1,6 +1,10 @@
 from shlex import quote
 
 
+defaults = {
+    'build-ci': {},
+}
+
 @metadata_reactor.provides(
     'users/build-ci/authorized_users',
     'sudoers/build-ci',
@@ -18,7 +22,7 @@ def ssh_keys(metadata):
         },
         'sudoers': {
             'build-ci': {
-                f"/usr/bin/chown -R build-ci\:{quote(ci['group'])} {quote(ci['path'])}"
+                f"/usr/bin/chown -R build-ci\\:{quote(ci['group'])} {quote(ci['path'])}"
                     for ci in metadata.get('build-ci').values()
             }
         },

bundles/crystal/metadata.py

@@ -1,10 +1,20 @@
+debian_version = min([node.os_version, (11,)])[0] # FIXME
+
 defaults = {
     'apt': {
         'packages': {
             'crystal': {},
         },
         'sources': {
-            'deb https://download.opensuse.org/repositories/devel:/languages:/crystal/Debian_{version}/ /',
+            'crystal': {
+                # https://software.opensuse.org/download.html?project=devel%3Alanguages%3Acrystal&package=crystal
+                'urls': {
+                    'http://download.opensuse.org/repositories/devel:/languages:/crystal/Debian_Testing/',
+                },
+                'suites': {
+                    '/',
+                },
+            },
         },
     },
 }

bundles/dovecot/files/dovecot.conf

@@ -6,7 +6,7 @@ ssl_cert = </var/lib/dehydrated/certs/${node.metadata.get('mailserver/hostname')
 ssl_key = </var/lib/dehydrated/certs/${node.metadata.get('mailserver/hostname')}/privkey.pem
 ssl_dh = </etc/dovecot/dhparam.pem
 ssl_client_ca_dir = /etc/ssl/certs
-mail_location = maildir:~
+mail_location = maildir:${node.metadata.get('mailserver/maildir')}/%u:INDEX=${node.metadata.get('mailserver/maildir')}/index/%u
 mail_plugins = fts fts_xapian
 
 namespace inbox {

bundles/dovecot/items.py

@@ -20,6 +20,10 @@ directories = {
         'owner': 'vmail',
         'group': 'vmail',
     },
+    '/var/vmail/index': {
+        'owner': 'vmail',
+        'group': 'vmail',
+    },
     '/var/vmail/sieve': {
         'owner': 'vmail',
         'group': 'vmail',

bundles/download-server/items.py

@@ -1,6 +0,0 @@
-# directories = {
-#     '/var/lib/downloads': {
-#         'owner': 'downloads',
-#         'group': 'www-data',
-#     }
-# }

bundles/freescout/README.md

@@ -0,0 +1,23 @@
+Pg Pass workaround: set manually:
+
+```
+root@freescout /ro psql freescout
+psql (15.6 (Debian 15.6-0+deb12u1))
+Type "help" for help.
+
+freescout=# \password freescout
+Enter new password for user "freescout":
+Enter it again:
+freescout=#
+\q
+```
+
+
+# problems
+
+# check if /opt/freescout/.env is resettet
+# ckeck `psql -h localhost -d freescout -U freescout -W`with pw from .env
+# chown -R www-data:www-data /opt/freescout
+# sudo su - www-data -c 'php /opt/freescout/artisan freescout:clear-cache' -s /bin/bash
+# javascript funny? `sudo su - www-data -c 'php /opt/freescout/artisan storage:link' -s /bin/bash`
+# benutzer bilder weg? aus dem backup holen: `/opt/freescout/.zfs/snapshot/zfs-auto-snap_hourly-2024-11-22-1700/storage/app/public/users` `./customers`

bundles/freescout/items.py

@@ -0,0 +1,66 @@
+# https://github.com/freescout-helpdesk/freescout/wiki/Installation-Guide
+run_as = repo.libs.tools.run_as
+php_version = node.metadata.get('php/version')
+
+
+directories = {
+    '/opt/freescout': {
+        'owner': 'www-data',
+        'group': 'www-data',
+        # chown -R www-data:www-data /opt/freescout
+    },
+}
+
+actions = {
+    # 'clone_freescout': {
+    #     'command': run_as('www-data', 'git clone https://github.com/freescout-helpdesk/freescout.git /opt/freescout'),
+    #     'unless': 'test -e /opt/freescout/.git',
+    #     'needs': [
+    #         'pkg_apt:git',
+    #         'directory:/opt/freescout',
+    #     ],
+    # },
+    # 'pull_freescout': {
+    #     'command': run_as('www-data', 'git -C /opt/freescout fetch origin dist && git -C /opt/freescout reset --hard origin/dist && git -C /opt/freescout clean -f'),
+    #     'unless': run_as('www-data', 'git -C /opt/freescout fetch origin && git -C /opt/freescout status -uno | grep -q "Your branch is up to date"'),
+    #     'needs': [
+    #         'action:clone_freescout',
+    #     ],
+    #     'triggers': [
+    #         'action:freescout_artisan_update',
+    #         f'svc_systemd:php{php_version}-fpm.service:restart',
+    #     ],
+    # },
+    # 'freescout_artisan_update': {
+    #     'command': run_as('www-data', 'php /opt/freescout/artisan freescout:after-app-update'),
+    #     'triggered': True,
+    #     'needs': [
+    #         f'svc_systemd:php{php_version}-fpm.service:restart',
+    #         'action:pull_freescout',
+    #     ],
+    # },
+}
+
+# svc_systemd = {
+#     f'freescout-cron.service': {},
+# }
+
+# files = {
+#     '/opt/freescout/.env': {
+#         # https://github.com/freescout-helpdesk/freescout/blob/dist/.env.example
+#         # Every time you are making changes in .env file, in order changes to take an effect you need to run:
+#         # ´sudo su - www-data -c 'php /opt/freescout/artisan freescout:clear-cache' -s /bin/bash´
+#         'owner': 'www-data',
+#         'content': '\n'.join(
+#             f'{k}={v}' for k, v in
+#                 sorted(node.metadata.get('freescout/env').items())
+#         ) + '\n',
+#         'needs': [
+#             'directory:/opt/freescout',
+#             'action:clone_freescout',
+#         ],
+#     },
+# }
+
+#sudo su - www-data -s /bin/bash -c 'php /opt/freescout/artisan freescout:create-user --role admin --firstName M --lastName W --email freescout@freibrief.net --password gyh.jzv2bnf6hvc.HKG --no-interaction'
+#sudo su - www-data -s /bin/bash -c 'php /opt/freescout/artisan freescout:create-user --role admin --firstName M --lastName W --email freescout@freibrief.net --password gyh.jzv2bnf6hvc.HKG --no-interaction'

bundles/freescout/metadata.py

@@ -0,0 +1,121 @@
+from base64 import b64decode
+
+# hash: SCRAM-SHA-256$4096:tQNfqQi7seqNDwJdHqCHbg==$r3ibECluHJaY6VRwpvPqrtCjgrEK7lAkgtUO8/tllTU=:+eeo4M0L2SowfyHFxT2FRqGzezve4ZOEocSIo11DATA=
+database_password = repo.vault.password_for(f'{node.name} postgresql freescout').value
+
+defaults = {
+    'apt': {
+        'packages': {
+            'git': {},
+            'php': {},
+            'php-pgsql': {},
+            'php-fpm': {},
+            'php-mbstring': {},
+            'php-xml': {},
+            'php-imap': {},
+            'php-zip': {},
+            'php-gd': {},
+            'php-curl': {},
+            'php-intl': {},
+        },
+    },
+    'freescout': {
+        'env': {
+            'APP_TIMEZONE': 'Europe/Berlin',
+            'DB_CONNECTION': 'pgsql',
+            'DB_HOST': '127.0.0.1',
+            'DB_PORT': '5432',
+            'DB_DATABASE': 'freescout',
+            'DB_USERNAME': 'freescout',
+            'DB_PASSWORD': database_password,
+            'APP_KEY': 'base64:' + repo.vault.random_bytes_as_base64_for(f'{node.name} freescout APP_KEY', length=32).value
+        },
+    },
+    'php': {
+        'php.ini': {
+            'cgi': {
+                'fix_pathinfo': '0',
+            },
+        },
+    },
+    'postgresql': {
+        'roles': {
+            'freescout': {
+                'password_hash': repo.libs.postgres.generate_scram_sha_256(
+                    database_password,
+                    b64decode(repo.vault.random_bytes_as_base64_for(f'{node.name} postgres freescout', length=16).value.encode()),
+                ),
+            },
+        },
+        'databases': {
+            'freescout': {
+                'owner': 'freescout',
+            },
+        },
+    },
+    # 'systemd': {
+    #     'units': {
+    #         f'freescout-cron.service': {
+    #             'Unit': {
+    #                 'Description': 'Freescout Cron',
+    #                 'After': 'network.target',
+    #             },
+    #             'Service': {
+    #                 'User': 'www-data',
+    #                 'Nice': 10,
+    #                 'ExecStart': f"/usr/bin/php /opt/freescout/artisan schedule:run"
+    #             },
+    #             'Install': {
+    #                 'WantedBy': {
+    #                     'multi-user.target'
+    #                 }
+    #             },
+    #         }
+    #     },
+    # },
+    'systemd-timers': {
+        'freescout-cron': {
+            'command': '/usr/bin/php /opt/freescout/artisan schedule:run',
+            'when': '*-*-* *:*:00',
+            'RuntimeMaxSec': '180',
+            'user': 'www-data',
+        },
+    },
+    'zfs': {
+        'datasets': {
+            'tank/freescout': {
+                'mountpoint': '/opt/freescout',
+            },
+        },
+    },
+}
+
+
+
+
+@metadata_reactor.provides(
+    'freescout/env/APP_URL',
+)
+def freescout(metadata):
+    return {
+        'freescout': {
+            'env': {
+                'APP_URL': 'https://' + metadata.get('freescout/domain') + '/',
+            },
+        },
+    }
+
+
+@metadata_reactor.provides(
+    'nginx/vhosts',
+)
+def nginx(metadata):
+    return {
+        'nginx': {
+            'vhosts': {
+                metadata.get('freescout/domain'): {
+                    'content': 'freescout/vhost.conf',
+                },
+            },
+        },
+    }

bundles/gcloud/metadata.py

@@ -8,7 +8,15 @@ defaults = {
             'python3-crcmod': {},
         },
         'sources': {
-            'deb https://packages.cloud.google.com/apt cloud-sdk main',
+            'google-cloud': {
+                'url': 'https://packages.cloud.google.com/apt/',
+                'suites': {
+                    'cloud-sdk',
+                },
+                'components': {
+                    'main',
+                },
+            },
         },
     },
 }

bundles/gitea/items.py

@@ -2,11 +2,14 @@ from os.path import join
 from bundlewrap.utils.dicts import merge_dict
 
 
-version = version=node.metadata.get('gitea/version')
+version = node.metadata.get('gitea/version')
+assert not version.startswith('v')
+arch = node.metadata.get('system/architecture')
 
 downloads['/usr/local/bin/gitea'] = {
-    'url': f'https://dl.gitea.io/gitea/{version}/gitea-{version}-linux-amd64',
-    'sha256': node.metadata.get('gitea/sha256'),
+    # https://forgejo.org/releases/
+    'url': f'https://codeberg.org/forgejo/forgejo/releases/download/v{version}/forgejo-{version}-linux-{arch}',
+    'sha256_url': '{url}.sha256',
     'triggers': {
         'svc_systemd:gitea:restart',
     },
@@ -45,6 +48,7 @@ files['/etc/gitea/app.ini'] = {
         ),
     ),
     'owner': 'git',
+    'mode': '0600',
     'context': node.metadata['gitea'],
     'triggers': {
         'svc_systemd:gitea:restart',

bundles/gitea/metadata.py

@@ -1,4 +1,4 @@
-database_password = repo.vault.password_for(f'{node.name} postgresql gitea')
+database_password = repo.vault.password_for(f'{node.name} postgresql gitea').value
 
 defaults = {
     'apt': {
@@ -11,7 +11,20 @@ defaults = {
         },
     },
     'gitea': {
-        'conf': {},
+        'conf': {
+            'DEFAULT': {
+                'WORK_PATH': '/var/lib/gitea',
+            },
+            'database': {
+                'DB_TYPE': 'postgres',
+                'HOST': 'localhost:5432',
+                'NAME': 'gitea',
+                'USER': 'gitea',
+                'PASSWD': database_password,
+                'SSL_MODE': 'disable',
+                'LOG_SQL': 'false',
+            },
+        },
     },
     'postgresql': {
         'roles': {
@@ -30,8 +43,7 @@ defaults = {
             'gitea.service': {
                 'Unit': {
                     'Description': 'gitea',
-                    'After': 'syslog.target',
-                    'After': 'network.target',
+                    'After': {'syslog.target', 'network.target'},
                     'Requires': 'postgresql.service',
                 },
                 'Service': {
@@ -84,15 +96,6 @@ def conf(metadata):
                     'INTERNAL_TOKEN': repo.vault.password_for(f'{node.name} gitea internal_token'),
                     'SECRET_KEY': repo.vault.password_for(f'{node.name} gitea security_secret_key'),
                 },
-                'database': {
-                    'DB_TYPE': 'postgres',
-                    'HOST': 'localhost:5432',
-                    'NAME': 'gitea',
-                    'USER': 'gitea',
-                    'PASSWD': database_password,
-                    'SSL_MODE': 'disable',
-                    'LOG_SQL': 'false',
-                },
                 'service': {
                     'NO_REPLY_ADDRESS': f'noreply.{domain}',
                 },
@@ -115,7 +118,7 @@ def nginx(metadata):
                     'content': 'nginx/proxy_pass.conf',
                     'context': {
                         'target': 'http://127.0.0.1:3500',
-                    }
+                    },
                 },
             },
         },

bundles/grafana/items.py

@@ -18,16 +18,17 @@ admin_password = node.metadata.get('grafana/config/security/admin_password')
 port = node.metadata.get('grafana/config/server/http_port')
 actions['reset_grafana_admin_password'] = {
     'command': f"grafana-cli admin reset-admin-password {quote(admin_password)}",
-    'unless': f"curl http://admin:{quote(admin_password)}@localhost:{port}/api/org",
+    'unless': f"sleep 5 && curl http://admin:{quote(admin_password)}@localhost:{port}/api/org --fail",
     'needs': [
         'svc_systemd:grafana-server',
     ],
 }
 
 directories = {
-    '/etc/grafana': {
-    },
+    '/etc/grafana': {},
     '/etc/grafana/provisioning': {
+        'owner': 'grafana',
+        'group': 'grafana',
     },
     '/etc/grafana/provisioning/datasources': {
         'purge': True,
@@ -35,8 +36,13 @@ directories = {
     '/etc/grafana/provisioning/dashboards': {
         'purge': True,
     },
-    '/var/lib/grafana': {},
+    '/var/lib/grafana': {
+        'owner': 'grafana',
+        'group': 'grafana',
+    },
     '/var/lib/grafana/dashboards': {
+        'owner': 'grafana',
+        'group': 'grafana',
         'purge': True,
         'triggers': [
             'svc_systemd:grafana-server:restart',
@@ -47,6 +53,8 @@ directories = {
 files = {
     '/etc/grafana/grafana.ini': {
         'content': repo.libs.ini.dumps(node.metadata.get('grafana/config')),
+        'owner': 'grafana',
+        'group': 'grafana',
         'triggers': [
             'svc_systemd:grafana-server:restart',
         ],
@@ -56,6 +64,8 @@ files = {
             'apiVersion': 1,
             'datasources': list(node.metadata.get('grafana/datasources').values()),
         }),
+        'owner': 'grafana',
+        'group': 'grafana',
         'triggers': [
             'svc_systemd:grafana-server:restart',
         ],
@@ -72,6 +82,8 @@ files = {
                 },
             }],
         }),
+        'owner': 'grafana',
+        'group': 'grafana',
         'triggers': [
             'svc_systemd:grafana-server:restart',
         ],

bundles/grafana/metadata.py

@@ -8,8 +8,19 @@ defaults = {
             'grafana': {},
         },
         'sources': {
-            'deb https://packages.grafana.com/oss/deb stable main',
+            'grafana': {
+                'urls': {
+                    'https://packages.grafana.com/oss/deb',
+                },
+                'suites': {
+                    'stable',
+                },
+                'components': {
+                    'main',
+                },
+            },
         },
+
     },
     'grafana': {
         'config': {

bundles/homeassistant-supervised/README.md

@@ -0,0 +1,23 @@
+https://github.com/home-assistant/supervised-installer?tab=readme-ov-file
+https://github.com/home-assistant/os-agent/tree/main?tab=readme-ov-file#using-home-assistant-supervised-on-debian
+https://docs.docker.com/engine/install/debian/
+
+
+
+
+
+https://www.home-assistant.io/installation/linux#install-home-assistant-supervised
+https://github.com/home-assistant/supervised-installer
+https://github.com/home-assistant/architecture/blob/master/adr/0014-home-assistant-supervised.md
+
+DATA_SHARE=/usr/share/hassio dpkg --force-confdef --force-confold -i homeassistant-supervised.deb
+
+neu debian
+ha installieren
+gucken ob geht
+dann bw drüberbügeln
+
+
+https://www.home-assistant.io/integrations/http/#ssl_certificate
+
+`wget "$(curl -L https://api.github.com/repos/home-assistant/supervised-installer/releases/latest | jq -r '.assets[0].browser_download_url')" -O homeassistant-supervised.deb && dpkg -i homeassistant-supervised.deb`

bundles/homeassistant-supervised/items.py

@@ -0,0 +1,30 @@
+from shlex import quote
+
+
+version = node.metadata.get('homeassistant/os_agent_version')
+
+directories = {
+    '/usr/share/hassio': {},
+}
+
+actions = {
+    'install_os_agent': {
+        'command': ' && '.join([
+            f'wget -O /tmp/os-agent.deb https://github.com/home-assistant/os-agent/releases/download/{quote(version)}/os-agent_{quote(version)}_linux_aarch64.deb',
+            'DEBIAN_FRONTEND=noninteractive dpkg -i /tmp/os-agent.deb',
+        ]),
+        'unless': f'test "$(apt -qq list os-agent | cut -d" " -f2)" = "{quote(version)}"',
+        'needs': {
+            'pkg_apt:',
+            'zfs_dataset:tank/homeassistant',
+        },
+    },
+    'install_homeassistant_supervised': {
+        'command': 'wget -O /tmp/homeassistant-supervised.deb https://github.com/home-assistant/supervised-installer/releases/latest/download/homeassistant-supervised.deb && apt install /tmp/homeassistant-supervised.deb',
+        'unless': 'apt -qq list homeassistant-supervised | grep -q "installed"',
+        'needs': {
+            'action:install_os_agent',
+        },
+    },
+}
+

bundles/homeassistant-supervised/metadata.py

@@ -0,0 +1,65 @@
+defaults = {
+    'apt': {
+        'packages': {
+            # homeassistant-supervised
+            'apparmor': {},
+            'bluez': {},
+            'cifs-utils': {},
+            'curl': {},
+            'dbus': {},
+            'jq': {},
+            'libglib2.0-bin': {},
+            'lsb-release': {},
+            'network-manager': {},
+            'nfs-common': {},
+            'systemd-journal-remote': {},
+            'systemd-resolved': {},
+            'udisks2': {},
+            'wget': {},
+            # docker
+            'docker-ce': {},
+            'docker-ce-cli': {},
+            'containerd.io': {},
+            'docker-buildx-plugin': {},
+            'docker-compose-plugin': {},
+       },
+        'sources': {
+            # docker: https://docs.docker.com/engine/install/debian/#install-using-the-repository
+            'docker': {
+                'urls': {
+                    'https://download.docker.com/linux/debian',
+                },
+                'suites': {
+                    '{codename}',
+                },
+                'components': {
+                    'stable',
+                },
+            },
+        },
+    },
+    'zfs': {
+        'datasets': {
+            'tank/homeassistant': {
+                'mountpoint': '/usr/share/hassio',
+                'needed_by': {
+                    'directory:/usr/share/hassio',
+                },
+            },
+        },
+    },
+}
+
+@metadata_reactor.provides(
+    'nginx/vhosts',
+)
+def nginx(metadata):
+    return {
+        'nginx': {
+            'vhosts': {
+                metadata.get('homeassistant/domain'): {
+                    'content': 'homeassistant/vhost.conf',
+                },
+            },
+        },
+    }

bundles/homeassistant/items.py

@@ -1,20 +0,0 @@
-users = {
-    'homeassistant': {
-        'home': '/var/lib/homeassistant',
-    },
-}
-
-directories = {
-    '/var/lib/homeassistant': {
-        'owner': 'homeassistant',
-    },
-    '/var/lib/homeassistant/config': {
-        'owner': 'homeassistant',
-    },
-    '/var/lib/homeassistant/venv': {
-        'owner': 'homeassistant',
-    },
-}
-
-
-# https://wiki.instar.com/de/Software/Linux/Home_Assistant/

bundles/homeassistant/metadata.py

@@ -1,20 +0,0 @@
-defaults = {
-    'apt': {
-        'packages': {
-            'python3': {},
-            'python3-dev': {},
-            'python3-pip': {},
-            'python3-venv': {},
-            'libffi-dev': {},
-            'libssl-dev': {},
-            'libjpeg-dev': {},
-            'zlib1g-dev': {},
-            'autoconf': {},
-            'build-essential': {},
-            'libopenjp2-7': {},
-            'libtiff5': {},
-            'libturbojpeg0-dev': {},
-            'tzdata': {},
-        },
-    },
-}

bundles/htop/files/htoprc.global

@@ -24,7 +24,7 @@ header_margin=1
 detailed_cpu_time=0
 cpu_count_from_one=1
 show_cpu_usage=0
-show_cpu_frequency=0
+show_cpu_frequency=1
 show_cpu_temperature=0
 degree_fahrenheit=0
 update_process_names=0

bundles/icinga2/files/conf.d/notifications.conf

@@ -13,9 +13,9 @@ apply Notification "mail-icingaadmin" to Host {
   user_groups = host.vars.notification.mail.groups
   users = host.vars.notification.mail.users
 
-  //interval = 2h
 
-  //vars.notification_logtosyslog = true
+
+
 
   assign where host.vars.notification.mail
 }
@@ -25,9 +25,9 @@ apply Notification "mail-icingaadmin" to Service {
   user_groups = host.vars.notification.mail.groups
   users = host.vars.notification.mail.users
 
-  //interval = 2h
 
-  //vars.notification_logtosyslog = true
+
+
 
   assign where host.vars.notification.mail
 }

bundles/icinga2/files/features/journald.conf

@@ -0,0 +1,7 @@
+/**
+ * The JournaldLogger type writes log information to the systemd journal.
+ */
+
+object JournaldLogger "journald" {
+    severity = "warning"
+}

bundles/icinga2/files/features/mainlog.conf

@@ -0,0 +1,4 @@
+/**
+ * This file is requires for inital apt install.
+ * The JournaldLogger type writes log information to the systemd journal.
+ */

bundles/icinga2/files/features/syslog.conf

@@ -1,3 +0,0 @@
-object SyslogLogger "syslog" {
-    severity = "warning"
-}

bundles/icinga2/items.py

@@ -10,6 +10,24 @@ directories = {
             'svc_systemd:icinga2.service:restart',
         ],
     },
+   '/etc/icinga2/pki': { # required for apt install
+       'purge': True,
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0750',
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+   '/etc/icinga2/zones.d': { # required for apt install
+       'purge': True,
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0750',
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
     '/etc/icinga2/conf.d': {
         'purge': True,
         'owner': 'nagios',
@@ -195,7 +213,8 @@ files = {
 # FEATURES
 
 for feature, context in {
-    'syslog': {},
+    'mainlog': {},
+#    'journald': {}, FIXME
     'notification': {},
     'checker': {},
     'api': {},
@@ -250,7 +269,7 @@ svc_systemd = {
     'icinga2.service': {
         'needs': [
             'pkg_apt:icinga2-ido-pgsql',
-            'svc_systemd:postgresql',
+            'svc_systemd:postgresql.service',
         ],
     },
 }

bundles/icinga2/metadata.py

@@ -9,7 +9,21 @@ defaults = {
             'monitoring-plugins': {},
         },
         'sources': {
-            'deb https://packages.icinga.com/debian icinga-{release} main',
+            'icinga': {
+                'types': {
+                    'deb',
+                    'deb-src',
+                },
+                'urls': {
+                    'https://packages.icinga.com/debian',
+                },
+                'suites': {
+                    'icinga-{codename}',
+                },
+                'components': {
+                    'main',
+                },
+            },
         },
     },
     'icinga2': {
@@ -49,7 +63,6 @@ defaults = {
                 'mountpoint': '/var/lib/icinga2',
                 'needed_by': {
                     'pkg_apt:icinga2',
-                    'pkg_apt:icingaweb2',
                     'pkg_apt:icinga2-ido-pgsql',
                 },
             },

bundles/icingaweb2/README.md

@@ -2,3 +2,4 @@
 - open /icingaweb2/setup in browser
 - fill in values from metadata
 - apply
+- make sure tls cert exists and is owned by nagios

bundles/icingaweb2/items.py

@@ -4,18 +4,27 @@ directories = {
         'owner': 'www-data',
         'group': 'icingaweb2',
         'mode': '2770',
+        'needs': [
+            'pkg_apt:icingaweb2',
+        ],
     },
     '/etc/icingaweb2/enabledModules': {
 #        'purge': True,
         'owner': 'www-data',
         'group': 'icingaweb2',
         'mode': '2770',
+        'needs': [
+            'pkg_apt:icingaweb2',
+        ],
     },
     '/etc/icingaweb2/modules': {
 #        'purge': True,
         'owner': 'www-data',
         'group': 'icingaweb2',
         'mode': '2770',
+        'needs': [
+            'pkg_apt:icingaweb2',
+        ],
     },
 }
 
@@ -25,6 +34,9 @@ files = {
         'owner': 'www-data',
         'group': 'icingaweb2',
         'mode': '0660',
+        'needs': [
+            'pkg_apt:icingaweb2',
+        ],
     },
 }
 
@@ -33,6 +45,9 @@ symlinks = {
         'target': '/usr/share/icingaweb2/modules/monitoring',
         'owner': 'www-data',
         'group': 'icingaweb2',
+        'needs': [
+            'pkg_apt:icingaweb2',
+        ],
     },
 }
 
@@ -48,6 +63,9 @@ for name in [
         'owner': 'www-data',
         'group': 'icingaweb2',
         'mode': '0660',
+        'needs': [
+            'pkg_apt:icingaweb2',
+        ],
     }
 
 for name in [
@@ -60,4 +78,7 @@ for name in [
         'owner': 'www-data',
         'group': 'icingaweb2',
         'mode': '0660',
+        'needs': [
+            'pkg_apt:icingaweb2',
+        ],
     }

bundles/icingaweb2/metadata.py

@@ -3,7 +3,6 @@ from hashlib import sha3_256
 defaults = {
     'apt': {
         'packages': {
-            'icingaweb2': {},
             'php-ldap': {},
             'php-json': {},
             'php-intl': {},
@@ -11,11 +10,25 @@ defaults = {
             'php-gd': {},
             'php-imagick': {},
             'php-pgsql': {},
-            'icingaweb2-module-monitoring': {},
+            'icingaweb2': {},
+            #'icingaweb2-module-monitoring': {}, # ?
         },
         'sources': {
-            'deb https://packages.icinga.com/debian icinga-{release} main',
-            'deb https://packages.icinga.com/debian icinga-{release}-snapshots main',
+            'icinga': {
+                'types': {
+                    'deb',
+                    'deb-src',
+                },
+                'urls': {
+                    'https://packages.icinga.com/debian',
+                },
+                'suites': {
+                    'icinga-{codename}',
+                },
+                'components': {
+                    'main',
+                },
+            },
         },
     },
     'icingaweb2': {
@@ -164,6 +177,7 @@ def nginx(metadata):
                 metadata.get('icingaweb2/hostname'): {
                     'content': 'icingaweb2/vhost.conf',
                     'context': {
+                        'php_version': metadata.get('php/version'),
                     },
                 },
             },

bundles/influxdb2/items.py

@@ -4,6 +4,7 @@ from shlex import quote
 directories['/var/lib/influxdb'] = {
     'owner': 'influxdb',
     'group': 'influxdb',
+    'mode': '0750',
     'needs': [
         'zfs_dataset:tank/influxdb',
     ],

bundles/influxdb2/metadata.py

@@ -7,7 +7,17 @@ defaults = {
             'influxdb2-cli': {},
         },
         'sources': {
-            'deb https://repos.influxdata.com/debian {release} stable',
+            'influxdata': {
+                'urls': {
+                    'https://repos.influxdata.com/debian',
+                },
+                'suites': {
+                    'stable',
+                },
+                'components': {
+                    'main',
+                },
+            },
         },
     },
     'nftables': {

bundles/java/metadata.py

@@ -19,7 +19,7 @@ def apt(metadata):
     return {
         'apt': {
             'packages': {
-                f'openjdk-{metadata.get("java/version")}-jre': {},
+                f'openjdk-{metadata.get("java/version")}-jre-headless': {},
             }
         }
     }

bundles/kea-dhcpd/items.py

@@ -0,0 +1,21 @@
+from json import dumps
+from bundlewrap.metadata import MetadataJSONEncoder
+
+files = {
+    '/etc/kea/kea-dhcp4.conf': {
+        'content': dumps(node.metadata.get('kea'), indent=4, sort_keys=True, cls=MetadataJSONEncoder),
+        'triggers': [
+            'svc_systemd:kea-dhcp4-server:restart',
+        ],
+    },
+}
+
+svc_systemd = {
+    'kea-dhcp4-server': {
+        'needs': [
+            'pkg_apt:kea-dhcp4-server',
+            'file:/etc/kea/kea-dhcp4.conf',
+            'svc_systemd:systemd-networkd:restart',
+        ],
+    },
+}

bundles/kea-dhcpd/metadata.py

@@ -0,0 +1,96 @@
+from ipaddress import ip_interface, ip_network
+
+hashable = repo.libs.hashable.hashable
+
+
+defaults = {
+    'apt': {
+        'packages': {
+            'kea-dhcp4-server': {},
+        },
+    },
+    'kea': {
+        'Dhcp4': {
+            'interfaces-config': {
+                'interfaces': set(),
+            },
+            'lease-database': {
+                'type': 'memfile',
+                'lfc-interval': 3600
+            },
+            'subnet4': set(),
+            'loggers': set([
+                hashable({
+                    'name': 'kea-dhcp4',
+                    'output_options': [
+                        {
+                            'output': 'syslog',
+                        }
+                    ],
+                    'severity': 'INFO',
+                }),
+            ]),
+        },
+    },
+}
+
+
+@metadata_reactor.provides(
+    'kea/Dhcp4/interfaces-config/interfaces',
+    'kea/Dhcp4/subnet4',
+)
+def subnets(metadata):
+    subnet4 = set()
+    interfaces = set()
+    reservations = set(
+        hashable({
+            'hw-address': network_conf['mac'],
+            'ip-address': str(ip_interface(network_conf['ipv4']).ip),
+        })
+            for other_node in repo.nodes
+            for network_conf in other_node.metadata.get('network', {}).values()
+            if 'mac' in network_conf
+    )
+
+    for network_name, network_conf in metadata.get('network').items():
+        dhcp_server_config = network_conf.get('dhcp_server_config', None)
+
+        if dhcp_server_config:
+            _network = ip_network(dhcp_server_config['subnet'])
+
+            subnet4.add(hashable({
+                'subnet': dhcp_server_config['subnet'],
+                'pools': [
+                    {
+                        'pool': f'{dhcp_server_config['pool_from']} - {dhcp_server_config['pool_to']}',
+                    },
+                ],
+                'option-data': [
+                    {
+                        'name': 'routers',
+                        'data': dhcp_server_config['router'],
+                    },
+                    {
+                        'name': 'domain-name-servers',
+                        'data': '10.0.10.2',
+                    },
+                ],
+                'reservations': set(
+                    reservation
+                        for reservation in reservations
+                        if ip_interface(reservation['ip-address']).ip in _network
+                ),
+            }))
+
+            interfaces.add(network_conf.get('interface', network_name))
+
+    return {
+        'kea': {
+            'Dhcp4': {
+                'interfaces-config': {
+                    'interfaces': interfaces,
+                },
+                'subnet4': subnet4,
+            },
+        },
+    }

bundles/left4dead2/files/server.cfg

@@ -1,34 +1,36 @@
 hostname "CroneKorkN : ${name}"
 sv_contact "admin@sublimity.de"
 
-// assign serevr to steam group
+
 sv_steamgroup "${','.join(steamgroups)}"
 
-// no annoying message of the day
+rcon_password "${rcon_password}"
+
+
 motd_enabled 0
 
-// enable cheats
+
 sv_cheats 1
 
-// allow inconsistent files on clients (weapon mods for example)
+
 sv_consistency 0
 
-// connect from internet
+
 sv_lan 0
 
-// join game at any point
+
 sv_allow_lobby_connect_only 0
 
-// allowed modes
+
 sv_gametypes "coop,realism,survival,versus,teamversus,scavenge,teamscavenge"
 
-// network
+
 sv_minrate 30000
 sv_maxrate 60000
 sv_mincmdrate 66
 sv_maxcmdrate 101
 
-// logging
+
 sv_logsdir "logs-${name}"        //Folder in the game directory where server logs will be stored.
 log on            //Creates a logfile (on | off)
 sv_logecho 0            //default 0; Echo log information to the console.

bundles/left4dead2/items.py

@@ -1,66 +1,22 @@
 assert node.has_bundle('steam') and node.has_bundle('steam-workshop-download')
 
 directories = {
-    '/opt/steam/left4dead2/left4dead2/ems/admin system': {
-        'owner': 'steam',
-        'group': 'steam',
-        'mode': '0744',
-    },
-    '/opt/steam/left4dead2/left4dead2/cfg/server': {
+    '/opt/steam/left4dead2-servers': {
         'owner': 'steam',
         'group': 'steam',
+        'mode': '0755',
         'purge': True,
     },
-    '/opt/steam/left4dead2/left4dead2/addons': {
+    # Current zfs doesnt support zfs upperdir. The support was added in October 2022. Move upperdir - unused anyway -
+    # to another dir. Also move workdir alongside it, as it has to be on same fs.
+    '/opt/steam-zfs-overlay-workarounds': {
         'owner': 'steam',
         'group': 'steam',
+        'mode': '0755',
         'purge': True,
     },
 }
 
-files = {
-    '/opt/steam/left4dead2/left4dead2/ems/admin system/admins.txt': {
-        'owner': 'steam',
-        'content': '\n'.join(sorted(node.metadata.get('left4dead2/admins'))),
-    }
-}
-
-for name, config in node.metadata.get('left4dead2/servers').items():
-    files[f'/opt/steam/left4dead2/left4dead2/cfg/server/{name}.cfg'] = {
-        'content_type': 'mako',
-        'source': 'server.cfg',
-        'context': {
-            'name': name,
-            'steamgroups': node.metadata.get('left4dead2/steamgroups'),
-        },
-        'owner': 'steam',
-        'group': 'steam',
-        'triggers': [
-            f'svc_systemd:left4dead2-{name}.service:restart',
-        ],
-    }
-    svc_systemd[f'left4dead2-{name}.service'] = {
-        'needs': [
-            f'file:/opt/steam/left4dead2/left4dead2/cfg/server/{name}.cfg',
-            f'file:/usr/local/lib/systemd/system/left4dead2-{name}.service',
-        ],
-    }
-
-
-files[f'/opt/steam/left4dead2/left4dead2/addons/readme.txt'] = {
-    'content_type': 'any',
-    'owner': 'steam',
-    'group': 'steam',
-}
-
-
-for id in node.metadata.get('left4dead2/workshop'):
-    files[f'/opt/steam/left4dead2/left4dead2/addons/{id}.vpk'] = {
-        'content_type': 'any',
-        'owner': 'steam',
-        'group': 'steam',
-    }
-
 # /opt/steam/steam/.steam/sdk32/steamclient.so: cannot open shared object file: No such file or directory
 symlinks = {
     '/opt/steam/steam/.steam/sdk32': {
@@ -69,3 +25,98 @@ symlinks = {
         'group': 'steam',
     }
 }
+
+#
+# SERVERS
+#
+
+for name, config in node.metadata.get('left4dead2/servers').items():
+
+    #overlay
+    directories[f'/opt/steam/left4dead2-servers/{name}'] = {
+        'owner': 'steam',
+        'group': 'steam',
+    }
+    directories[f'/opt/steam-zfs-overlay-workarounds/{name}/upper'] = {
+        'owner': 'steam',
+        'group': 'steam',
+    }
+    directories[f'/opt/steam-zfs-overlay-workarounds/{name}/workdir'] = {
+        'owner': 'steam',
+        'group': 'steam',
+    }
+
+    # conf
+    files[f'/opt/steam/left4dead2-servers/{name}/left4dead2/cfg/server.cfg'] = {
+        'content_type': 'mako',
+        'source': 'server.cfg',
+        'context': {
+            'name': name,
+            'steamgroups': node.metadata.get('left4dead2/steamgroups'),
+            'rcon_password': config['rcon_password'],
+        },
+        'owner': 'steam',
+        'group': 'steam',
+        'triggers': [
+            f'svc_systemd:left4dead2-{name}.service:restart',
+        ],
+    }
+
+    # service
+    svc_systemd[f'left4dead2-{name}.service'] = {
+        'needs': [
+            f'file:/opt/steam/left4dead2-servers/{name}/left4dead2/cfg/server.cfg',
+            f'file:/usr/local/lib/systemd/system/left4dead2-{name}.service',
+        ],
+    }
+
+    #
+    # ADDONS
+    #
+
+    # base
+    files[f'/opt/steam/left4dead2-servers/{name}/left4dead2/addons/readme.txt'] = {
+        'content_type': 'any',
+        'owner': 'steam',
+        'group': 'steam',
+    }
+    directories[f'/opt/steam/left4dead2-servers/{name}/left4dead2/addons'] = {
+        'owner': 'steam',
+        'group': 'steam',
+        'purge': True,
+        'triggers': [
+            f'svc_systemd:left4dead2-{name}.service:restart',
+        ],
+    }
+    for id in [
+        *config.get('workshop', []),
+        *node.metadata.get('left4dead2/workshop'),
+    ]:
+        files[f'/opt/steam/left4dead2-servers/{name}/left4dead2/addons/{id}.vpk'] = {
+            'content_type': 'any',
+            'owner': 'steam',
+            'group': 'steam',
+            'triggers': [
+                f'svc_systemd:left4dead2-{name}.service:restart',
+            ],
+        }
+
+    # admin system
+
+    directories[f'/opt/steam/left4dead2-servers/{name}/left4dead2/ems/admin system'] = {
+        'owner': 'steam',
+        'group': 'steam',
+        'mode': '0755',
+        'triggers': [
+            f'svc_systemd:left4dead2-{name}.service:restart',
+        ],
+    }
+    files[f'/opt/steam/left4dead2-servers/{name}/left4dead2/ems/admin system/admins.txt'] = {
+        'owner': 'steam',
+        'group': 'steam',
+        'mode': '0755',
+        'content': '\n'.join(sorted(node.metadata.get('left4dead2/admins'))),
+        'triggers': [
+            f'svc_systemd:left4dead2-{name}.service:restart',
+        ],
+    }

bundles/left4dead2/metadata.py

@@ -17,40 +17,85 @@ defaults = {
 
 
 @metadata_reactor.provides(
-    'steam-workshop-download/left4dead',
+    'left4dead2/servers',
 )
-def workshop_download(metadata):
-    if not metadata.get('left4dead2/workshop'):
-        return {}
-
+def rconn_password(metadata):
+    # only works from localhost!
     return {
-        'steam-workshop-download': {
-            'left4dead': {
-                'ids': metadata.get('left4dead2/workshop'),
-                'path': '/opt/steam/left4dead2/left4dead2/addons',
-                'user': 'steam',
+        'left4dead2': {
+            'servers': {
+                server: {
+                    'rcon_password': repo.vault.password_for(f'{node.name} left4dead2 {server} rcon', length=24),
+                }
+                    for server in metadata.get('left4dead2/servers')
             },
         },
     }
 
 
 @metadata_reactor.provides(
+    'steam-workshop-download',
     'systemd/units',
 )
 def server_units(metadata):
     units = {}
+    workshop = {}
 
     for name, config in metadata.get('left4dead2/servers').items():
+        # mount overlay
+        mountpoint = f'/opt/steam/left4dead2-servers/{name}'
+        mount_unit_name = mountpoint[1:].replace('-', '\\x2d').replace('/', '-') + '.mount'
+        units[mount_unit_name] = {
+            'Unit': {
+                'Description': f"Mount left4dead2 server {name} overlay",
+                'Conflicts': {'umount.target'},
+                'Before': {'umount.target'},
+            },
+            'Mount': {
+                'What': 'overlay',
+                'Where': mountpoint,
+                'Type': 'overlay',
+                'Options': ','.join([
+                    'auto',
+                    'lowerdir=/opt/steam/left4dead2',
+                    f'upperdir=/opt/steam-zfs-overlay-workarounds/{name}/upper',
+                    f'workdir=/opt/steam-zfs-overlay-workarounds/{name}/workdir',
+                ]),
+            },
+            'Install': {
+                'RequiredBy': {
+                    f'left4dead2-{name}.service',
+                },
+            },
+        }
+
+        # individual workshop
+        workshop_ids = config.get('workshop', set()) | metadata.get('left4dead2/workshop', set())
+        if  workshop_ids:
+            workshop[f'left4dead2-{name}'] = {
+                'ids': workshop_ids,
+                'path': f'/opt/steam/left4dead2-servers/{name}/left4dead2/addons',
+                'user': 'steam',
+                'requires': {
+                    mount_unit_name,
+                },
+                'required_by': {
+                    f'left4dead2-{name}.service',
+                },
+            }
+
+        # left4dead2 server unit
         units[f'left4dead2-{name}.service'] = {
             'Unit': {
                 'Description': f'left4dead2 server {name}',
-                'After': {'steam.target'},
+                'After': {'steam-update.service'},
+                'Requires': {'steam-update.service'},
             },
             'Service': {
                 'User': 'steam',
                 'Group': 'steam',
-                'WorkingDirectory': '/opt/steam/left4dead2',
-                'ExecStart': f'/opt/steam/left4dead2/srcds_run -port {config["port"]} +exec server/{name}.cfg',
+                'WorkingDirectory': f'/opt/steam/left4dead2-servers/{name}',
+                'ExecStart': f'/opt/steam/left4dead2-servers/{name}/srcds_run -port {config["port"]} +exec server.cfg',
                 'Restart': 'on-failure',
             },
             'Install': {
@@ -59,6 +104,7 @@ def server_units(metadata):
         }
 
     return {
+        'steam-workshop-download': workshop,
         'systemd': {
             'units': units,
         },
@@ -74,8 +120,8 @@ def firewall(metadata):
     return {
         'nftables': {
             'input': {
-                f"tcp dport {{ {', '.join(ports)} }} accept",
-                f"udp dport {{ {', '.join(ports)} }} accept",
+                f"tcp dport {{ {', '.join(sorted(ports))} }} accept",
+                f"udp dport {{ {', '.join(sorted(ports))} }} accept",
             },
         },
     }

bundles/letsencrypt/README.md

@@ -1,6 +1,6 @@
 https://github.com/dehydrated-io/dehydrated/wiki/example-dns-01-nsupdate-script
 
-```
+```sh
 printf "server 127.0.0.1
 zone acme.resolver.name.
 update add _acme-challenge.ckn.li.acme.resolver.name. 600 IN TXT "hello"

bundles/letsencrypt/files/hook.sh

@@ -4,7 +4,7 @@ set -o pipefail
 
 deploy_challenge() {
   echo "
-    server 10.0.11.3
+    server ${server}
     zone ${zone}.
     update add $1.${zone}. 60 IN TXT \"$3\"
     send
@@ -13,7 +13,7 @@ deploy_challenge() {
 
 clean_challenge() {
   echo "
-    server 10.0.11.3
+    server ${server}
     zone ${zone}.
     update delete $1.${zone}. TXT
     send

bundles/letsencrypt/items.py

@@ -56,6 +56,7 @@ for domain in node.metadata.get('letsencrypt/domains').keys():
         'unless': f'/etc/dehydrated/letsencrypt-ensure-some-certificate {domain} true',
         'needs': {
             'file:/etc/dehydrated/letsencrypt-ensure-some-certificate',
+            'pkg_apt:dehydrated',
         },
         'needed_by': {
             'svc_systemd:nginx',

bundles/linux/items.py

@@ -0,0 +1,41 @@
+from shlex import quote
+
+def generate_sysctl_key_value_pairs_from_json(json_data, parents=[]):
+    if isinstance(json_data, dict):
+        for key, value in json_data.items():
+            yield from generate_sysctl_key_value_pairs_from_json(value, [*parents, key])
+    elif isinstance(json_data, list):
+        raise ValueError(f"List not supported: '{json_data}'")
+    else:
+        # If it's a leaf node, yield the path
+        yield (parents, json_data)
+
+key_value_pairs = generate_sysctl_key_value_pairs_from_json(node.metadata.get('sysctl'))
+
+
+files= {
+    '/etc/sysctl.conf': {
+        'content': '\n'.join(
+            sorted(
+                f"{'.'.join(path)}={value}"
+                    for path, value in key_value_pairs
+            ),
+        ),
+        'triggers': [
+            'svc_systemd:systemd-sysctl.service:restart',
+        ],
+    },
+}
+
+svc_systemd = {
+    'systemd-sysctl.service': {},
+}
+
+for path, value in key_value_pairs:
+    actions[f'reload_sysctl.conf_{path}'] = {
+        'command': f"sysctl --values {'.'.join(path)}  | grep -q {quote('^'+value+'$')}",
+        'needs': [
+            f'action:systemd-sysctl.service',
+            f'action:systemd-sysctl.service:restart',
+        ],
+    }

bundles/linux/metadata.py

@@ -0,0 +1,3 @@
+defaults = {
+    'sysctl': {},
+}

bundles/locale/items.py

@@ -20,18 +20,19 @@ files = {
 }
 
 actions = {
+    'systemd-locale': {
+        'command': f'localectl set-locale LANG="{default_locale}"',
+        'unless': f'localectl | grep -Fi "system locale" | grep -Fi "{default_locale}"',
+        'triggers': {
+            'action:locale-gen',
+        },
+    },
     'locale-gen': {
         'command': 'locale-gen',
         'triggered': True,
         'needs': {
             'pkg_apt:locales',
-        },
-    },
-    'systemd-locale': {
-        'command': f'localectl set-locale LANG="{default_locale}"',
-        'unless': f'localectl | grep -Fi "system locale" | grep -Fi "{default_locale}"',
-        'preceded_by': {
-            'action:locale-gen',
+            'action:systemd-locale',
         },
     },
 }

bundles/macbook/files/bundlewrap

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+cd "$OLDPWD"
+
+export BW_ITEM_WORKERS=$(expr "$(sysctl -n hw.logicalcpu)" '*' 12 '/' 10)
+export BW_NODE_WORKERS=$(expr 320 '/' "$BW_ITEM_WORKERS")

bundles/macbook/files/gnu

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+cd "$OLDPWD"
+
+PATH_add "/opt/homebrew/opt/gnu-sed/libexec/gnubin"
+PATH_add "/opt/homebrew/opt/grep/libexec/gnubin"

bundles/macbook/files/macbook-update

@@ -0,0 +1,46 @@
+#!/bin/bash -l
+
+sudo tee /etc/pam.d/sudo << EOT
+# sudo: auth account password session
+auth       sufficient     pam_tid.so
+auth       sufficient     pam_smartcard.so
+auth       required       pam_opendirectory.so
+account    required       pam_permit.so
+password   required       pam_deny.so
+session    required       pam_permit.so
+EOT
+
+sudo xcodebuild -license accept
+xcode-select --install
+
+git -C ~/.zsh/oh-my-zsh pull
+
+brew upgrade
+brew upgrade --cask --greedy
+
+pyenv install --skip-existing
+
+sudo softwareupdate -ia --verbose
+
+if test "$(defaults read com.apple.dock autohide-time-modifier)" == 0.16
+then
+    defaults write com.apple.dock autohide-time-modifier -float 0.16
+    RESTART_DOCK=TRUE
+fi
+
+if test "$(defaults read com.apple.dock autohide-delay)" -ne 0
+then
+    defaults write com.apple.dock autohide-delay -float 0
+    RESTART_DOCK=TRUE
+fi
+
+if test "$RESTART_DOCK" = TRUE
+then
+    killall Dock
+fi
+
+sudo systemsetup -setremotelogin on # enable ssh
+
+pip install --upgrade pip
+
+# https://sysadmin-journal.com/apache-directory-studio-on-the-apple-m1/

bundles/macbook/files/pyenv

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+cd "$OLDPWD"
+
+if test -f .venv/bin/python && test "$(realpath .venv/bin/python)" != "$(realpath "$(pyenv which python)")"
+then
+  echo "rebuilding venv für new python version"
+  rm -rf .venv .pip_upgrade_timestamp
+fi

bundles/macbook/files/ssh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+cd "$OLDPWD"

bundles/macbook/files/venv

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+cd "$OLDPWD"
+
+python3 -m venv .venv
+source .venv/bin/activate
+PATH_add .venv/bin
+
+NOW=$(date +%s)
+if test -e .pip_upgrade_timestamp
+then
+  LAST=$(cat .pip_upgrade_timestamp)
+else
+  LAST=0
+fi
+DELTA=$(expr "$NOW" - "$LAST")
+echo "last pip upgrade $DELTA seconds ago"
+if test "$DELTA" -gt 86400
+then
+  python3 -m pip --require-virtualenv install pip wheel --upgrade
+  python3 -m pip --require-virtualenv install -r requirements.txt --upgrade
+  if test -e optional-requirements.txt
+  then
+    python3 -m pip --require-virtualenv install -r optional-requirements.txt --upgrade
+  fi
+  date +%s > .pip_upgrade_timestamp
+fi

bundles/macbook/files/zshrc

@@ -0,0 +1,33 @@
+export PATH=~/.bin:$PATH
+export PATH=~/.cargo/bin:$PATH
+
+export ZSH=~/.zsh/oh-my-zsh
+export ZSH_HOSTNAME='sm'
+ZSH_THEME="bw"
+HIST_STAMPS="yyyy/mm/dd"
+plugins=(
+  zsh-autosuggestions
+  git
+)
+source $ZSH/oh-my-zsh.sh
+
+ulimit -S -n 24000
+
+antivir() {
+    printf 'scanning for viruses' && sleep 1 && printf '.' && sleep 1 && printf '.' && sleep 1 && printf '.' &&
+sleep 1 && echo '\nyour computer is safe!'
+}
+
+eval "$(rbenv init -)"
+eval "$(pyenv init -)"
+eval "$(direnv hook zsh)"
+eval "$(op completion zsh)"; compdef _op op
+
+# //S/M
+
+sshn() {
+    ssh "$(tr '.' ' ' <<< "$1" | tac -s ' ' | xargs | tr ' ' '.').smhss.de"
+}
+pingn() {
+    ping "$(tr '.' ' ' <<< "$1" | tac -s ' ' | xargs | tr ' ' '.').smhss.de"
+}

bundles/macbook/items.py

@@ -0,0 +1,47 @@
+# brew install
+
+actions['brew_install'] = {
+    'command': '/opt/homebrew/bin/brew install ' + ' '.join(node.metadata.get('brew')),
+    'unless': f"""PKGS=$(/opt/homebrew/bin/brew leaves); for p in {' '.join(node.metadata.get('brew'))}; do grep -q "$p" <<< $PKGS || exit 9; done"""
+}
+
+# bw init
+
+directories['/Users/mwiegand/.config/bundlewrap/lock'] = {}
+
+# home
+
+files['/Users/mwiegand/.zshrc'] = {
+    'source': 'zshrc',
+    'mode': '0644',
+}
+
+# updater
+
+files['/Users/mwiegand/.bin/macbook-update'] = {
+    'mode': '755',
+}
+
+with open(f'{repo.path}/bundles/zsh/files/bw.zsh-theme') as f:
+    files['/Users/mwiegand/.zsh/oh-my-zsh/themes/bw.zsh-theme'] = {
+        'content': f.read(),
+        'mode': '0644',
+    }
+
+# direnv
+
+directories['/Users/mwiegand/.local/share/direnv'] = {}
+files['/Users/mwiegand/.local/share/direnv/gnu'] = {}
+files['/Users/mwiegand/.local/share/direnv/pyenv'] = {}
+files['/Users/mwiegand/.local/share/direnv/venv'] = {}
+files['/Users/mwiegand/.local/share/direnv/bundlewrap'] = {}
+
+
+##################
+
+for element in [*files.values(), *directories.values()]:
+    element.update({
+        'owner': 'mwiegand',
+        'group': 'staff',
+        **element,
+    })

bundles/macbook/metadata.py

@@ -0,0 +1,3 @@
+defaults = {
+    'brew': {},
+}

bundles/mailserver-autoconfig/files/autodiscover.php

@@ -1,6 +1,6 @@
 <?php
 
-// https://raw.githubusercontent.com/Radiergummi/autodiscover/master/autodiscover/autodiscover.php
+
 
 /********************************
  * Autodiscover responder
@@ -8,45 +8,45 @@
  * This PHP script is intended to respond to any request to http(s)://mydomain.com/autodiscover/autodiscover.xml.
  * If configured properly, it will send a spec-complient autodiscover XML response, pointing mail clients to the
  * appropriate mail services.
- * If you use MAPI or ActiveSync, stick with the Autodiscover service your mail server provides for you. But if 
+ * If you use MAPI or ActiveSync, stick with the Autodiscover service your mail server provides for you. But if
  * you use POP/IMAP servers, this will provide autoconfiguration to Outlook, Apple Mail and mobile devices.
  *
- * To work properly, you'll need to set the service (sub)domains below in the settings section to the correct 
+ * To work properly, you'll need to set the service (sub)domains below in the settings section to the correct
  * domain names, adjust ports and SSL.
  */
 
-//get raw POST data so we can extract the email address
+
 $request = file_get_contents("php://input");
 
-// optional debug log
+
 # file_put_contents( 'request.log', $request, FILE_APPEND );
 
-// retrieve email address from client request
+
 preg_match( "/\<EMailAddress\>(.*?)\<\/EMailAddress\>/", $request, $email );
 
-// check for invalid mail, to prevent XSS
+
 if (filter_var($email[1], FILTER_VALIDATE_EMAIL) === false) {
 	throw new Exception('Invalid E-Mail provided');
 }
 
-// get domain from email address
+
 $domain = substr( strrchr( $email[1], "@" ), 1 );
 
 /**************************************
  *   Port and server settings below   *
  **************************************/
 
-// IMAP settings
+
 $imapServer = 'imap.' . $domain; // imap.example.com
 $imapPort   = 993;
 $imapSSL    = true;
 
-// SMTP settings
+
 $smtpServer = 'smtp.' . $domain; // smtp.example.com
 $smtpPort   = 587;
 $smtpSSL    = true;
 
-//set Content-Type
+
 header( 'Content-Type: application/xml' );
 ?>
 <?php echo '<?xml version="1.0" encoding="utf-8" ?>'; ?>

bundles/mailserver/metadata.py

@@ -33,6 +33,12 @@ defaults = {
                 'mountpoint': '/var/vmail',
                 'compression': 'on',
             },
+            'tank/vmail/index': {
+                'mountpoint': '/var/vmail/index',
+                'compression': 'on',
+                'com.sun:auto-snapshot': 'false',
+                'backup': False,
+            },
         },
     },
 }
@@ -43,12 +49,30 @@ defaults = {
 )
 def dns(metadata):
     dns = {}
-    
+
     for domain in metadata.get('mailserver/domains'):
         dns[domain] = {
             'MX': [f"5 {metadata.get('mailserver/hostname')}."],
             'TXT': ['v=spf1 a mx -all'],
         }
+        report_email = metadata.get('mailserver/dmarc_report_email')
+        dns[f'_dmarc.{domain}'] = {
+            'TXT': ['; '.join(f'{k}={v}' for k, v in {
+                # dmarc version
+                'v': 'DMARC1',
+                # reject on failure
+                'p': 'reject',
+                # standard reports
+                'rua': f'mailto:{report_email}',
+                # forensic reports
+                'fo': 1,
+                'ruf': f'mailto:{report_email}',
+                # require alignment between the DKIM domain and the parent Header From domain
+                'adkim': 's',
+                # require alignment between the SPF domain (the sender) and the Header From domain
+                'aspf': 's',
+            }.items())]
+        }
 
     return {
         'dns': dns,
@@ -66,4 +90,4 @@ def letsencrypt(metadata):
                 },
             },
         },
-    } 
+    }

bundles/mariadb/README.md

@@ -0,0 +1 @@
+https://mariadb.com/kb/en/systemd/#configuring-mariadb-to-write-the-error-log-to-syslog

bundles/mariadb/files/override.conf

@@ -0,0 +1,11 @@
+% for section, options in sorted(conf.items()):
+[${section}]
+% for key, value in sorted(options.items()):
+% if value is None:
+${key}
+% else:
+${key} = ${value}
+% endif
+% endfor
+
+% endfor

bundles/mariadb/items.py

@@ -0,0 +1,91 @@
+from shlex import quote
+
+def mariadb(sql, **kwargs):
+    kwargs_string = ''.join(f" --{k} {v}" for k, v in kwargs.items())
+    return f"mariadb{kwargs_string} -Bsr --execute {quote(sql)}"
+
+directories = {
+    '/var/lib/mysql': {
+        'owner': 'mysql',
+        'group': 'mysql',
+        'needs': [
+            'zfs_dataset:tank/mariadb',
+        ],
+        'needed_by': [
+            'pkg_apt:mariadb-server',
+            'pkg_apt:mariadb-client',
+        ],
+    },
+}
+
+files = {
+    '/etc/mysql/conf.d/override.conf': {
+        'context': {
+            'conf': node.metadata.get('mariadb/conf'),
+        },
+        'content_type': 'mako',
+    },
+}
+
+svc_systemd = {
+    'mariadb.service': {
+        'needs': [
+            'pkg_apt:mariadb-server',
+            'pkg_apt:mariadb-client',
+        ],
+    },
+}
+
+actions = {
+    'mariadb_sec_remove_anonymous_users': {
+        'command': mariadb("DELETE FROM mysql.global_priv WHERE User=''"),
+        'unless': mariadb("SELECT count(0) FROM mysql.global_priv WHERE User = ''") + " | grep -q '^0$'",
+        'needs': [
+            'svc_systemd:mariadb.service',
+        ],
+        'triggers': [
+            'svc_systemd:mariadb.service:restart',
+        ],
+    },
+    'mariadb_sec_remove_remote_root': {
+        'command': mariadb("DELETE FROM mysql.global_priv WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1')"),
+        'unless': mariadb("SELECT count(0) FROM mysql.global_priv WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1')") + " | grep -q '^0$'",
+        'needs': [
+            'svc_systemd:mariadb.service',
+        ],
+        'triggers': [
+            'svc_systemd:mariadb.service:restart',
+        ],
+    },
+}
+
+for db, conf in node.metadata.get('mariadb/databases', {}).items():
+    actions[f'mariadb_create_database_{db}'] = {
+        'command': mariadb(f"CREATE DATABASE {db}"),
+        'unless': mariadb(f"SHOW DATABASES LIKE '{db}'") + f" | grep -q '^{db}$'",
+        'needs': [
+            'svc_systemd:mariadb.service',
+        ],
+    }
+    actions[f'mariadb_user_{db}_create'] = {
+        'command': mariadb(f"CREATE USER {db}"),
+        'unless': mariadb(f"SELECT User FROM mysql.user WHERE User = '{db}'") + f" | grep -q '^{db}$'",
+        'needs': [
+            f'action:mariadb_create_database_{db}',
+        ],
+    }
+    pw = conf['password']
+    actions[f'mariadb_user_{db}_password'] = {
+        'command': mariadb(f"SET PASSWORD FOR {db} = PASSWORD('{conf['password']}')"),
+        'unless': f'echo {quote(pw)} | mariadb -u {db} -e quit -p',
+        'needs': [
+            f'action:mariadb_user_{db}_create',
+        ],
+    }
+    actions[f'mariadb_grant_privileges_to_{db}'] = {
+        'command': mariadb(f"GRANT ALL PRIVILEGES ON {db}.* TO '{db}'", database=db),
+        'unless': mariadb(f"SHOW GRANTS FOR {db}") + f" | grep -q '^GRANT ALL PRIVILEGES ON `{db}`.* TO `{db}`@`%`'",
+        'needs': [
+            f'action:mariadb_user_{db}_create',
+        ],
+    }

bundles/mariadb/metadata.py

@@ -0,0 +1,45 @@
+defaults = {
+    'apt': {
+        'packages': {
+            'mariadb-server': {
+                'needs': {
+                    'zfs_dataset:tank/mariadb',
+                },
+            },
+            'mariadb-client': {
+                'needs': {
+                    'zfs_dataset:tank/mariadb',
+                },
+            },
+        },
+    },
+    'mariadb': {
+        'databases': {},
+        'conf': {
+            # https://www.reddit.com/r/zfs/comments/u1xklc/mariadbmysql_database_settings_for_zfs
+            'mysqld': {
+                'skip-innodb_doublewrite': None,
+                'innodb_flush_method': 'fsync',
+                'innodb_doublewrite': '0',
+                'innodb_use_atomic_writes': '0',
+                'innodb_use_native_aio': '0',
+                'innodb_read_io_threads': '10',
+                'innodb_write_io_threads': '10',
+                'innodb_buffer_pool_size': '26G',
+                'innodb_flush_log_at_trx_commit': '1',
+                'innodb_log_file_size': '1G',
+                'innodb_flush_neighbors': '0',
+                'innodb_fast_shutdown': '2',
+            },
+        },
+    },
+    'zfs': {
+        'datasets': {
+            'tank/mariadb': {
+                'mountpoint': '/var/lib/mysql',
+                'recordsize': '16384',
+                'atime': 'off',
+            },
+        },
+    },
+}

bundles/monitored/files/downtime

@@ -79,6 +79,7 @@ if action == 'add':
             'end_time': now + duration_seconds,
             'type': 'Host',
             'child_options': 'DowntimeTriggeredChildren',
+            'all_services': True,
             'filter':  f'match("{node_name}", host.name)',
             'pretty': True,
         }

bundles/network/metadata.py

@@ -5,38 +5,89 @@ defaults = {
 }
 
 
+@metadata_reactor.provides(
+    'network',
+)
+def dhcp(metadata):
+    networks = {}
+
+    for network_name, network_conf in metadata.get('network').items():
+        _interface = ip_interface(network_conf['ipv4'])
+        _ip = _interface.ip
+        _network = _interface.network
+        _hosts = list(_network.hosts())
+
+        if network_conf.get('dhcp_server', False):
+            networks[network_name] = {
+                'dhcp_server_config': {
+                    'subnet': str(_network),
+                    'pool_from': str(_hosts[len(_hosts)//2]),
+                    'pool_to': str(_hosts[-3]),
+                    'router': str(_ip),
+                    'domain-name-servers': str(_ip),
+                }
+            }
+    return {
+        'network': networks,
+    }
+
+
 @metadata_reactor.provides(
     'systemd/units',
 )
 def units(metadata):
     units = {}
 
-    for type, network in metadata.get('network').items():
-        units[f'{type}.network'] = {
+    for network_name, network_conf in metadata.get('network').items():
+        interface_type = network_conf.get('type', None)
+
+        # network
+
+        units[f'{network_name}.network'] = {
             'Match': {
-                'Name': network['interface'],
+                'Name': network_name if interface_type == 'vlan' else network_conf['interface'],
             },
             'Network': {
-                'DHCP': 'no',
-                'IPv6AcceptRA': 'no',
+                'DHCP': network_conf.get('dhcp', 'no'),
+                'IPv6AcceptRA': network_conf.get('dhcp', 'no'),
+                'VLAN': set(network_conf.get('vlans', set()))
             }
         }
 
+        # type
+
+        if interface_type:
+            units[f'{network_name}.network']['Match']['Type'] = interface_type
+
+        # ips
+
         for i in [4, 6]:
-            if network.get(f'ipv{i}', None):
-                units[f'{type}.network'].update({
+            if network_conf.get(f'ipv{i}', None):
+                units[f'{network_name}.network'].update({
                     f'Address#ipv{i}': {
-                        'Address': network[f'ipv{i}'],
+                        'Address': network_conf[f'ipv{i}'],
                     },
                 })
-                if f'gateway{i}' in network:
-                    units[f'{type}.network'].update({
+                if f'gateway{i}' in network_conf:
+                    units[f'{network_name}.network'].update({
                         f'Route#ipv{i}': {
-                            'Gateway': network[f'gateway{i}'],
+                            'Gateway': network_conf[f'gateway{i}'],
                             'GatewayOnlink': 'yes',
                         }
                     })
 
+        # as vlan
+
+        if interface_type == 'vlan':
+            units[f"{network_name}.netdev"] = {
+                'NetDev': {
+                    'Name': network_name,
+                    'Kind': 'vlan',
+                },
+                'VLAN': {
+                    'Id': network_conf['id'],
+                }
+            }
 
     return {
         'systemd': {

bundles/nextcloud-picsort/files/nextcloud-picsort

@@ -72,7 +72,7 @@ if [ "$SCAN" == "TRUE" ]; then
   sudo -u www-data php /opt/nextcloud/occ files:scan --path "$REL_SOURCE_PATH"
   sudo -u www-data php /opt/nextcloud/occ files:scan --path "$REL_UNSORTABLE_PATH"
   sudo -u www-data php /opt/nextcloud/occ files:scan --path "$REL_DEST_PATH"
-  sudo -u www-data php /opt/nextcloud/occ preview:pre-generate
+  #sudo -u www-data php /opt/nextcloud/occ preview:pre-generate
 fi
 
 echo "FINISH."

bundles/nextcloud/files/managed.config.php

@@ -1,43 +1,3 @@
 <?php
 # https://docs.nextcloud.com/server/stable/admin_manual/configuration_server/config_sample_php_parameters.html#multiple-config-php-file
-$CONFIG = array (
-  "dbuser" => "nextcloud",
-  "dbpassword" => "${db_password}",
-  "dbname" => "nextcloud",
-  "dbhost" => "localhost",
-  "dbtype" => "pgsql",
-  "datadirectory" => "/var/lib/nextcloud",
-  "dbport" => "5432",
-  "apps_paths" => [
-    [
-      "path"     => "/opt/nextcloud/apps",
-      "url"      => "/apps",
-      "writable" => false,
-    ],
-    [
-      "path"     => "/var/lib/nextcloud/.userapps",
-      "url"      => "/userapps",
-      "writable" => true,
-    ],
-  ],
-  "cache_path" => "/var/lib/nextcloud/.cache",
-  "upgrade.disable-web" => true,
-  "memcache.local" => "\\OC\\Memcache\\Redis",
-  "memcache.locking" => "\\OC\\Memcache\\Redis",
-  "memcache.distributed" => "\OC\Memcache\Redis",
-  "redis" => [
-    "host" => "/var/run/redis/nextcloud.sock",
-  ],
-  'trusted_domains' =>
-  array (
-    0 => 'localhost',
-    1 => '127.0.0.1',
-    2 => '${hostname}',
-  ),
-  "log_type" => "syslog",
-  "syslog_tag" => "nextcloud",
-  "logfile" => "",
-  "loglevel" => 3,
-  "default_phone_region" => "DE",
-  "versions_retention_obligation" => "auto, 90",
-);
+$CONFIG = json_decode(file_get_contents("/etc/nextcloud/managed.config.json"), $associative = true);

bundles/nextcloud/items.py

@@ -1,8 +1,7 @@
-assert node.has_bundle('php')
+import json
 
-from shlex import quote
-from os.path import join
-from mako.template import Template
+
+assert node.has_bundle('php')
 
 version = node.metadata.get('nextcloud/version')
 
@@ -69,19 +68,22 @@ symlinks = {
 
 files = {
     '/etc/nextcloud/managed.config.php': {
-        'content_type': 'mako',
         'owner': 'www-data',
         'group': 'www-data',
         'mode': '640',
-        'context': {
-            'db_password': node.metadata.get('postgresql/roles/nextcloud/password'),
-            'hostname': node.metadata.get('nextcloud/hostname'),
-        },
         'needs': [
             'directory:/etc/nextcloud',
         ],
     },
-}
+    '/etc/nextcloud/managed.config.json': {
+        'content': json.dumps(node.metadata.get('nextcloud/config'), indent=4, sort_keys=True),
+        'owner': 'www-data',
+        'group': 'www-data',
+        'mode': '640',
+        'needs': [
+            'directory:/etc/nextcloud',
+        ],
+    },}
 
 # SETUP
 
@@ -124,7 +126,7 @@ files['/opt/nextcloud_upgrade_status.php'] = {
         'action:extract_nextcloud',
     ],
 }
-    
+
 actions['upgrade_nextcloud'] = {
     'command': repo.libs.nextcloud.occ('upgrade'),
     'unless': 'sudo -u www-data php /opt/nextcloud_upgrade_status.php; test $? -ne 99',

bundles/nextcloud/metadata.py

@@ -16,16 +16,12 @@ defaults = {
             'php-cgi': {},
             'php-zip': {},
             'php-pgsql': {},
-            'php-bz2': {}, # face recognition
             'php-intl': {},
             'php-imagick': {},
             'libmagickcore-6.q16-6-extra': {},
             'php-gmp': {},
             'php-bcmath': {},
         },
-        'sources': {
-            'deb https://repo.delellis.com.ar {release} {release}', # face recognition
-        },
     },
     'archive': {
         'paths': {
@@ -33,8 +29,8 @@ defaults = {
                 'exclude': [
                     '^appdata_',
                     '^updater-',
-                    '^nextcloud\.log',
-                    '^updater\.log',
+                    '^nextcloud\\.log',
+                    '^updater\\.log',
                     '^[^/]+/cache',
                     '^[^/]+/files_versions',
                     '^[^/]+/files_trashbin',
@@ -49,7 +45,8 @@ defaults = {
     },
     'nextcloud': {
         'admin_user': 'admin',
-        'admin_pass': repo.vault.password_for(f'{node.name} nextcloud admin pw'),
+        'admin_pass': repo.vault.password_for(f'{node.name} nextcloud admin pw').value,
+        'config': {},
     },
     'php': {
         'post_max_size': '32G',
@@ -69,7 +66,7 @@ defaults = {
     'postgresql': {
         'roles': {
             'nextcloud': {
-                'password': repo.vault.password_for(f'{node.name} nextcloud db pw'),
+                'password': repo.vault.password_for(f'{node.name} nextcloud db pw').value,
             },
         },
         'databases': {
@@ -86,21 +83,71 @@ defaults = {
             'command': '/usr/bin/php -f /opt/nextcloud/cron.php',
             'when': '*:0/5',
             'user': 'www-data',
+            'kill_mode': 'process',
         },
         'nextcloud-rescan': {
             'command': '/opt/nextcloud_rescan',
             'when': 'Sun 00:00:00',
             'user': 'www-data',
         },
-        'nextcloud-face-recognition': {
-            'command': '/usr/bin/php -f /opt/nextcloud/occ face:background_job -t 1800',
-            'when': '*:0/10',
-            'user': 'www-data',
-        },
     },
 }
 
 
+@metadata_reactor.provides(
+    'nextcloud/config',
+)
+def config(metadata):
+    return {
+        'nextcloud': {
+            'config': {
+                'dbuser': 'nextcloud',
+                'dbpassword': metadata.get('postgresql/roles/nextcloud/password'),
+                'dbname': 'nextcloud',
+                'dbhost': 'localhost',
+                'dbtype': 'pgsql',
+                'dbtableprefix': 'oc_',
+                'datadirectory': '/var/lib/nextcloud',
+                'dbport': '5432',
+                'apps_paths': [
+                    {
+                        'path': '/opt/nextcloud/apps',
+                        'url': '/apps',
+                        'writable': False
+                    },
+                    {
+                        'path': '/var/lib/nextcloud/.userapps',
+                        'url': '/userapps',
+                        'writable': True
+                    }
+                ],
+                'cache_path': '/var/lib/nextcloud/.cache',
+                'upgrade.disable-web': True,
+                'memcache.local': '\\OC\\Memcache\\Redis',
+                'memcache.locking': '\\OC\\Memcache\\Redis',
+                'memcache.distributed': '\\OC\\Memcache\\Redis',
+                'redis': {
+                    'host': '/var/run/redis/nextcloud.sock'
+                },
+                'trusted_domains': [
+                    'localhost',
+                    '127.0.0.1',
+                    metadata.get('nextcloud/hostname'),
+                ],
+                'log_type': 'syslog',
+                'syslog_tag': 'nextcloud',
+                'logfile': '',
+                'loglevel': 3,
+                'default_phone_region': 'DE',
+                'versions_retention_obligation': 'auto, 90',
+                'simpleSignUpLink.shown': False,
+                'allow_local_remote_servers': True, # FIXME?
+                'maintenance_window_start': 1, # https://docs.nextcloud.com/server/29/admin_manual/configuration_server/background_jobs_configuration.html#maintenance-window-start
+            },
+        },
+    }
+
+
 @metadata_reactor.provides(
     'zfs/datasets',
 )
@@ -135,16 +182,3 @@ def vhost(metadata):
             },
         },
     }
-
-
-@metadata_reactor.provides(
-    'nginx/vhosts'
-)
-def pdlib(metadata):
-    return {
-        'apt': {
-            'packages': {
-                f"php{metadata.get('php/version')}-pdlib": {}, # face recognition
-            },
-        },
-    }

bundles/nftables/README.md

@@ -0,0 +1,11 @@
+NFT
+===
+
+examples
+--------
+
+### iperf3
+
+```sh
+nft add rule inet filter input tcp dport 5201 accept
+```

bundles/nginx-rtmps/README.md

@@ -0,0 +1,17 @@
+OBS
+===
+
+https://serverfault.com/a/1019344/88708
+
+publish
+-------
+
+obs stream to `rtmps://rtmp.sublimity.de:1936/24b626n246n2462v46bb2n2` stream key `ckn`
+
+input
+-----
+
+- Media Source
+- uncheck Local File
+- Input: `rtmps://rtmp.sublimity.de:1936/24b626n246n2462v46bb2n2/ckn`
+

bundles/nginx-rtmps/files/rtmps.conf

@@ -0,0 +1,38 @@
+stream {
+    upstream rtmp {
+        server 127.0.0.1:1935;
+    }
+
+    server {
+        listen 1936 ssl;
+        listen [::]:1936 ssl;
+
+        ssl_certificate /var/lib/dehydrated/certs/${server_name}/fullchain.pem;
+        ssl_certificate_key /var/lib/dehydrated/certs/${server_name}/privkey.pem;
+
+        proxy_pass rtmp;
+    }
+}
+
+rtmp {
+    server {
+        listen 127.0.0.1:1935;
+        chunk_size 4096;
+
+        application ${stream_key} {
+            live on;
+            record off;
+            allow publish 127.0.0.1;  # for streaming through nginx
+            allow play 127.0.0.1;     # for the pull from /live
+        }
+
+        application live {
+            live on;
+            record off;
+            deny publish all;         # no need to publish on /live
+            allow play all;           # playing allowed
+
+            pull rtmp://127.0.0.1:1935/${stream_key};
+        }
+    }
+}

bundles/nginx-rtmps/items.py

@@ -0,0 +1,12 @@
+files = {
+    '/etc/nginx/conf.d/rtmps.conf': {
+        'content_type': 'mako',
+        'context': {
+            'server_name': node.metadata.get('nginx-rtmps/hostname'),
+            'stream_key': node.metadata.get('nginx-rtmps/stream_key'),
+        },
+        'triggers': [
+            'svc_systemd:nginx:restart',
+        ],
+    },
+}

bundles/nginx-rtmps/metadata.py

@@ -0,0 +1,56 @@
+defaults = {
+    'apt': {
+        'packages': {
+            'libnginx-mod-stream': {},
+            'libnginx-mod-rtmp': {},
+        },
+    },
+    'nftables': {
+        'input': {
+            'tcp dport 1936 accept',
+        },
+    },
+    'nginx': {
+        'modules': {
+            'rtmp',
+            'stream',
+        },
+    },
+}
+
+
+@metadata_reactor.provides(
+    'nginx-rtmps/stream_key',
+)
+def stream_key(metadata):
+    return {
+        'nginx-rtmps': {
+            'stream_key': repo.vault.password_for(f"{metadata.get('id')} nginx-rtmps stream_key", length=24)
+        },
+    }
+
+
+@metadata_reactor.provides(
+    'dns',
+)
+def dns(metadata):
+    return {
+        'dns': {
+            metadata.get('nginx-rtmps/hostname'): repo.libs.ip.get_a_records(metadata),
+        }
+    }
+
+
+@metadata_reactor.provides(
+    'letsencrypt/domains',
+)
+def letsencrypt(metadata):
+    return {
+        'letsencrypt': {
+            'domains': {
+                metadata.get('nginx-rtmps/hostname'): {
+                    'reload': {'nginx'},
+                },
+            },
+        },
+    }

bundles/nginx/files/fastcgi

@@ -17,10 +17,12 @@ fastcgi_param  SERVER_SOFTWARE    nginx/$nginx_version;
 
 fastcgi_param  REMOTE_ADDR        $remote_addr;
 fastcgi_param  REMOTE_PORT        $remote_port;
+fastcgi_param  REMOTE_USER        $remote_user;
 fastcgi_param  SERVER_ADDR        $server_addr;
 fastcgi_param  SERVER_PORT        $server_port;
 fastcgi_param  SERVER_NAME        $server_name;
 
+# PHP only, required if PHP was built with --enable-force-cgi-redirect
 fastcgi_param  REDIRECT_STATUS    200;
 
 # This is the only thing that's different to the debian default.

bundles/nginx/files/mime.types

@@ -0,0 +1,95 @@
+types {
+    text/html                             html htm shtml;
+    text/css                              css;
+    text/xml                              xml;
+    image/gif                             gif;
+    image/jpeg                            jpeg jpg;
+    application/javascript                js;
+    application/atom+xml                  atom;
+    application/rss+xml                   rss;
+
+    text/mathml                           mml;
+    text/plain                            txt;
+    text/vnd.sun.j2me.app-descriptor      jad;
+    text/vnd.wap.wml                      wml;
+    text/x-component                      htc;
+
+    image/avif                            avif;
+    image/png                             png;
+    image/svg+xml                         svg svgz;
+    image/tiff                            tif tiff;
+    image/vnd.wap.wbmp                    wbmp;
+    image/webp                            webp;
+    image/x-icon                          ico;
+    image/x-jng                           jng;
+    image/x-ms-bmp                        bmp;
+
+    font/woff                             woff;
+    font/woff2                            woff2;
+
+    application/java-archive              jar war ear;
+    application/json                      json;
+    application/mac-binhex40              hqx;
+    application/msword                    doc;
+    application/pdf                       pdf;
+    application/postscript                ps eps ai;
+    application/rtf                       rtf;
+    application/vnd.apple.mpegurl         m3u8;
+    application/vnd.google-earth.kml+xml  kml;
+    application/vnd.google-earth.kmz      kmz;
+    application/vnd.ms-excel              xls;
+    application/vnd.ms-fontobject         eot;
+    application/vnd.ms-powerpoint         ppt;
+    application/vnd.oasis.opendocument.graphics        odg;
+    application/vnd.oasis.opendocument.presentation    odp;
+    application/vnd.oasis.opendocument.spreadsheet     ods;
+    application/vnd.oasis.opendocument.text            odt;
+    application/vnd.openxmlformats-officedocument.presentationml.presentation    pptx;
+    application/vnd.openxmlformats-officedocument.spreadsheetml.sheet    xlsx;
+    application/vnd.openxmlformats-officedocument.wordprocessingml.document    docx;
+    application/vnd.wap.wmlc              wmlc;
+    application/wasm                      wasm;
+    application/x-7z-compressed           7z;
+    application/x-cocoa                   cco;
+    application/x-java-archive-diff       jardiff;
+    application/x-java-jnlp-file          jnlp;
+    application/x-makeself                run;
+    application/x-perl                    pl pm;
+    application/x-pilot                   prc pdb;
+    application/x-rar-compressed          rar;
+    application/x-redhat-package-manager  rpm;
+    application/x-sea                     sea;
+    application/x-shockwave-flash         swf;
+    application/x-stuffit                 sit;
+    application/x-tcl                     tcl tk;
+    application/x-x509-ca-cert            der pem crt;
+    application/x-xpinstall               xpi;
+    application/xhtml+xml                 xhtml;
+    application/xspf+xml                  xspf;
+    application/zip                       zip;
+
+    application/octet-stream              bin exe dll;
+    application/octet-stream              deb;
+    application/octet-stream              dmg;
+    application/octet-stream              iso img;
+    application/octet-stream              msi msp msm;
+
+    audio/midi                            mid midi kar;
+    audio/mpeg                            mp3;
+    audio/ogg                             ogg;
+    audio/x-m4a                           m4a;
+    audio/x-realaudio                     ra;
+
+    video/3gpp                            3gpp 3gp;
+    video/mp2t                            ts;
+    video/mp4                             mp4;
+    video/mpeg                            mpeg mpg;
+    video/quicktime                       mov;
+    video/webm                            webm;
+    video/x-flv                           flv;
+    video/x-m4v                           m4v;
+    video/x-mng                           mng;
+    video/x-ms-asf                        asx asf;
+    video/x-ms-wmv                        wmv;
+    video/x-msvideo                       avi;
+}

bundles/nginx/files/nginx.conf

@@ -1,6 +1,12 @@
 pid /var/run/nginx.pid;
 user www-data;
-worker_processes 10;
+worker_processes ${worker_processes};
+
+% for module in sorted(modules):
+load_module modules/ngx_${module}_module.so;
+% endfor
+
+include /etc/nginx/conf.d/*;
 
 events {
     worker_connections 768;
@@ -15,7 +21,10 @@ http {
     server_names_hash_bucket_size 128;
     tcp_nopush on;
     client_max_body_size 32G;
-    
+    ssl_dhparam "/etc/ssl/certs/dhparam.pem";
+    # dont show nginx version
+    server_tokens off;
+
     % if node.has_bundle('php'):
     upstream php-handler {
         server unix:/var/run/php/php${node.metadata.get('php/version')}-fpm.sock;

bundles/nginx/files/proxy

@@ -0,0 +1,4 @@
+proxy_set_header Host $http_host;
+proxy_set_header X-Real-IP $remote_addr;
+proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+proxy_set_header X-Forwarded-Proto $scheme;

bundles/nginx/files/scgi

@@ -0,0 +1,16 @@
+scgi_param  REQUEST_METHOD     $request_method;
+scgi_param  REQUEST_URI        $request_uri;
+scgi_param  QUERY_STRING       $query_string;
+scgi_param  CONTENT_TYPE       $content_type;
+
+scgi_param  DOCUMENT_URI       $document_uri;
+scgi_param  DOCUMENT_ROOT      $document_root;
+scgi_param  SCGI               1;
+scgi_param  SERVER_PROTOCOL    $server_protocol;
+scgi_param  REQUEST_SCHEME     $scheme;
+scgi_param  HTTPS              $https if_not_empty;
+
+scgi_param  REMOTE_ADDR        $remote_addr;
+scgi_param  REMOTE_PORT        $remote_port;
+scgi_param  SERVER_PORT        $server_port;
+scgi_param  SERVER_NAME        $server_name;

bundles/nginx/files/uwsgi

@@ -0,0 +1,16 @@
+uwsgi_param  QUERY_STRING       $query_string;
+uwsgi_param  REQUEST_METHOD     $request_method;
+uwsgi_param  CONTENT_TYPE       $content_type;
+uwsgi_param  CONTENT_LENGTH     $content_length;
+
+uwsgi_param  REQUEST_URI        $request_uri;
+uwsgi_param  PATH_INFO          $document_uri;
+uwsgi_param  DOCUMENT_ROOT      $document_root;
+uwsgi_param  SERVER_PROTOCOL    $server_protocol;
+uwsgi_param  REQUEST_SCHEME     $scheme;
+uwsgi_param  HTTPS              $https if_not_empty;
+
+uwsgi_param  REMOTE_ADDR        $remote_addr;
+uwsgi_param  REMOTE_PORT        $remote_port;
+uwsgi_param  SERVER_PORT        $server_port;
+uwsgi_param  SERVER_NAME        $server_name;

bundles/nginx/items.py

@@ -3,19 +3,26 @@ from mako.template import Template
 from os.path import join
 
 directories = {
+    '/etc/nginx': {
+        'purge': True,
+        'triggers': {
+            'svc_systemd:nginx:restart',
+        },
+    },
     '/etc/nginx/sites': {
         'purge': True,
         'triggers': {
             'svc_systemd:nginx:restart',
         },
     },
-    '/etc/nginx/ssl': {
+    '/etc/nginx/params': {
         'purge': True,
         'triggers': {
             'svc_systemd:nginx:restart',
         },
     },
     '/var/www': {
+        'purge': True,
         'owner': 'www-data',
     },
 }
@@ -23,42 +30,54 @@ directories = {
 files = {
     '/etc/nginx/nginx.conf': {
         'content_type': 'mako',
+        'context': {
+            'modules': node.metadata.get('nginx/modules'),
+            'worker_processes': node.metadata.get('vm/cores'),
+        },
         'triggers': {
             'svc_systemd:nginx:restart',
         },
-    }, 
-    '/etc/nginx/fastcgi.conf': {
+    },
+    '/etc/nginx/params/fastcgi': {
         'triggers': {
             'svc_systemd:nginx:restart',
         },
-    }, 
+    },
+    '/etc/nginx/params/proxy': {
+        'triggers': {
+            'svc_systemd:nginx:restart',
+        },
+    },
+    '/etc/nginx/params/uwsgi': {
+        'triggers': {
+            'svc_systemd:nginx:restart',
+        },
+    },
+    '/etc/nginx/params/scgi': {
+        'triggers': {
+            'svc_systemd:nginx:restart',
+        },
+    },
+    '/etc/nginx/mime.types': {
+        'triggers': {
+            'svc_systemd:nginx:restart',
+        },
+    },
     '/etc/nginx/sites/80.conf': {
         'triggers': {
             'svc_systemd:nginx:restart',
         },
-    }, 
+    },
     '/etc/nginx/sites/stub_status.conf': {
         'triggers': {
             'svc_systemd:nginx:restart',
         },
-    }, 
-    '/etc/nginx/sites-available': {
-        'delete': True,
-        'needs': {
-            'pkg_apt:nginx',
-        },
-    },
-    '/etc/nginx/sites-enabled': {
-        'delete': True,
-        'needs': {
-            'pkg_apt:nginx',
-        },
     },
 }
 
 actions = {
     'nginx-generate-dhparam': {
-        'command': 'openssl dhparam -out /etc/ssl/certs/dhparam.pem 2048',
+        'command': 'openssl dhparam -dsaparam -out /etc/ssl/certs/dhparam.pem 4096',
         'unless': 'test -f /etc/ssl/certs/dhparam.pem',
     },
 }
@@ -88,7 +107,7 @@ for name, config in node.metadata.get('nginx/vhosts').items():
             'svc_systemd:nginx:restart',
         },
     }
-    
+
     if name in node.metadata.get('letsencrypt/domains'):
         files[f'/etc/nginx/sites/{name}']['needs'].append(
             f'action:letsencrypt_ensure-some-certificate_{name}',

bundles/nginx/metadata.py

@@ -1,11 +1,13 @@
 from shlex import quote
-from ipaddress import ip_interface
 
 
 defaults = {
     'apt': {
         'packages': {
             'nginx': {},
+            'apache2': {
+                'installed': False,
+            },
         },
     },
     'nftables': {
@@ -14,45 +16,21 @@ defaults = {
         },
     },
     'nginx': {
-        'vhosts': {
-            # '80': {
-            #     'content': 'nginx/80.conf',
-            # },
-            # 'stub_status': {
-            #     'content': 'nginx/stub_status.conf',
-            # },
-        },
-        'includes': {},
+        'vhosts': {},
+        'modules': set(),
     },
     'systemd': {
         'units': {
             'nginx.service.d/override.conf': {
                 'Unit': {
                     'After': {'network-online.target'},
+                    'Wants': {'network-online.target'},
                 },
             },
         },
     },
 }
 
-@metadata_reactor.provides(
-    'nginx/includes',
-)
-def includes(metadata):
-    return {
-        'nginx': {
-            'includes': {
-                'php': {
-                    'location ~ \.php$': {
-                        'include': 'fastcgi.conf',
-                        'fastcgi_split_path_info': '^(.+\.php)(/.+)$',
-                        'fastcgi_pass': f"unix:/run/php/php{metadata.get('php/version')}-fpm.sock",
-                    },
-                },
-            },
-        },
-    }
-
 
 @metadata_reactor.provides(
     'nginx/vhosts',
@@ -95,7 +73,6 @@ def dns(metadata):
 
 @metadata_reactor.provides(
     'letsencrypt/domains',
-    'letsencrypt/reload_after',
 )
 def letsencrypt(metadata):
     return {
@@ -124,3 +101,17 @@ def monitoring(metadata):
             },
         },
     }
+
+
+@metadata_reactor.provides(
+    'apt/packages',
+)
+def modules(metadata):
+    return {
+        'apt': {
+            'packages': {
+                f'libnginx-mod-{module}': {}
+                    for module in metadata.get('nginx/modules')
+            },
+        },
+    }

bundles/nodejs/README.md

@@ -0,0 +1,10 @@
+````python
+{
+    'metadata': {
+        'nodejs': {
+            'version': 16,
+        },
+    },
+}
+
+```

bundles/nodejs/items.py

@@ -0,0 +1,8 @@
+for package, conf in node.metadata.get('npm').items():
+    actions[f'npm_{package}'] = {
+        'command': f'npm install -g {package}',
+        'unless': f"npm list -g -json | jq -r '.dependencies | keys[]' | grep -q '^{package}$'",
+        'needs': [
+            'pkg_apt:nodejs',
+        ],
+    }

bundles/nodejs/metadata.py

@@ -0,0 +1,41 @@
+defaults = {
+    'apt': {
+        'packages': {
+            'nodejs': {},
+            # http://www.alvinsim.com/which-yarn/
+            'cmdtest': {
+                'installed': False,
+            },
+        },
+    },
+    'npm': {
+        'yarn': {},
+    },
+}
+
+
+@metadata_reactor.provides(
+    'apt/sources',
+)
+def sources(metadata):
+    version = metadata.get('nodejs/version')
+
+    return {
+        'apt': {
+            'sources': {
+                'nodesource': {
+                    'types': {
+                        'deb',
+                        'deb-src',
+                    },
+                    'url': 'https://deb.nodesource.com/node_{version}.x',
+                    'suites': {
+                        '{codename}',
+                    },
+                    'components': {
+                        'main',
+                    },
+                },
+            },
+        },
+    }

bundles/openhab/metadata.py

@@ -9,7 +9,17 @@ defaults = {
             },
         },
         'sources': {
-            'deb https://openhab.jfrog.io/artifactory/openhab-linuxpkg stable main',
+            'jfrog': {
+                'urls': {
+                    'https://openhab.jfrog.io/artifactory/openhab-linuxpkg',
+                },
+                'suites': {
+                    'stable',
+                },
+                'components': {
+                    'main',
+                },
+            },
         },
     },
     'zfs': {

bundles/php/items.py

@@ -1,9 +1,3 @@
-from os.path import join
-import json
-
-from bundlewrap.utils.dicts import merge_dict
-
-
 version = node.metadata.get('php/version')
 
 files = {
@@ -21,7 +15,7 @@ files = {
             f'pkg_apt:php{version}-fpm',
         },
         'triggers': {
-            f'svc_systemd:php{version}-fpm:restart',
+            f'svc_systemd:php{version}-fpm.service:restart',
         },
     },
     f'/etc/php/{version}/fpm/pool.d/www.conf': {
@@ -33,13 +27,13 @@ files = {
             f'pkg_apt:php{version}-fpm',
         },
         'triggers': {
-            f'svc_systemd:php{version}-fpm:restart',
+            f'svc_systemd:php{version}-fpm.service:restart',
         },
     },
 }
 
 svc_systemd = {
-    f'php{version}-fpm': {
+    f'php{version}-fpm.service': {
         'needs': {
             'pkg_apt:',
             f'file:/etc/php/{version}/fpm/php.ini',