wip

Reviewed-on: #19

bundles/apt/README.md

@@ -13,6 +13,9 @@
                     'deb',
                     'deb-src',
                 },
+                'options': { # optional
+                    'aarch': 'amd64',
+                },
                 'urls': {
                     'https://deb.debian.org/debian',
                 },

bundles/apt/items.py

@@ -62,6 +62,7 @@ files = {
     '/usr/lib/nagios/plugins/check_apt_upgradable': {
         'mode': '0755',
     },
+    # /etc/kernel/postinst.d/apt-auto-removal
 }
 
 actions = {

bundles/gitea/files/app.ini

@@ -40,7 +40,7 @@ ENABLE_OPENID_SIGNUP = false
 [service]
 REGISTER_EMAIL_CONFIRM = true
 ENABLE_NOTIFY_MAIL = true
-DISABLE_REGISTRATION = false
+DISABLE_REGISTRATION = true
 ALLOW_ONLY_EXTERNAL_REGISTRATION = false
 ENABLE_CAPTCHA = false
 REQUIRE_SIGNIN_VIEW = false

bundles/grafana/metadata.py

@@ -69,6 +69,9 @@ defaults = {
             },
         },
     },
+    'nginx': {
+        'has_websockets': True,
+    },
 }
 
 
@@ -144,6 +147,7 @@ def dns(metadata):
 def nginx(metadata):
     return {
         'nginx': {
+            'has_websockets': True,
             'vhosts': {
                 metadata.get('grafana/hostname'): {
                     'content': 'grafana/vhost.conf',

bundles/kea-dhcpd/items.py

@@ -15,7 +15,7 @@ svc_systemd = {
         'needs': [
             'pkg_apt:kea-dhcp4-server',
             'file:/etc/kea/kea-dhcp4.conf',
-            'svc_systemd:systemd-networkd:restart',
+            'svc_systemd:systemd-networkd.service:restart',
         ],
     },
 }

bundles/mariadb/items.py

@@ -11,7 +11,7 @@ directories = {
         'needs': [
             'zfs_dataset:tank/mariadb',
         ],
-        'needed_by': [
+        'needs': [
             'pkg_apt:mariadb-server',
             'pkg_apt:mariadb-client',
         ],

bundles/mariadb/metadata.py

@@ -3,12 +3,12 @@ defaults = {
         'packages': {
             'mariadb-server': {
                 'needs': {
-                    'zfs_dataset:tank/mariadb',
+                    #'zfs_dataset:tank/mariadb',
                 },
             },
             'mariadb-client': {
                 'needs': {
-                    'zfs_dataset:tank/mariadb',
+                    #'zfs_dataset:tank/mariadb',
                 },
             },
         },

bundles/nginx/files/nginx.conf

@@ -31,5 +31,13 @@ http {
     }
 
     % endif
-    include /etc/nginx/sites/*;
+
+    % if has_websockets:
+    map $http_upgrade $connection_upgrade {
+        default upgrade;
+        '' close;
+    }
+    % endif
+
+    include /etc/nginx/sites-enabled/*;
 }

bundles/nginx/items.py

@@ -9,7 +9,7 @@ directories = {
             'svc_systemd:nginx:restart',
         },
     },
-    '/etc/nginx/sites': {
+    '/etc/nginx/sites-available': {
         'purge': True,
         'triggers': {
             'svc_systemd:nginx:restart',
@@ -33,6 +33,7 @@ files = {
         'context': {
             'modules': node.metadata.get('nginx/modules'),
             'worker_processes': node.metadata.get('vm/cores'),
+            'has_websockets': node.metadata.get('nginx/has_websockets'),
         },
         'triggers': {
             'svc_systemd:nginx:restart',
@@ -75,6 +76,12 @@ files = {
     },
 }
 
+symlinks = {
+    '/etc/nginx/sites-enabled': {
+        'target': '/etc/nginx/sites-available',
+    },
+}
+
 actions = {
     'nginx-generate-dhparam': {
         'command': 'openssl dhparam -dsaparam -out /etc/ssl/certs/dhparam.pem 4096',
@@ -93,7 +100,7 @@ svc_systemd = {
 
 
 for name, config in node.metadata.get('nginx/vhosts').items():
-    files[f'/etc/nginx/sites/{name}'] = {
+    files[f'/etc/nginx/sites-available/{name}'] = {
         'content': Template(filename=join(repo.path, 'data', config['content'])).render(
             server_name=name,
             **config.get('context', {}),
@@ -109,6 +116,6 @@ for name, config in node.metadata.get('nginx/vhosts').items():
     }
 
     if name in node.metadata.get('letsencrypt/domains'):
-        files[f'/etc/nginx/sites/{name}']['needs'].append(
+        files[f'/etc/nginx/sites-available/{name}']['needs'].append(
             f'action:letsencrypt_ensure-some-certificate_{name}',
         )

bundles/nginx/metadata.py

@@ -18,6 +18,7 @@ defaults = {
     'nginx': {
         'vhosts': {},
         'modules': set(),
+        'has_websockets': False,
     },
     'systemd': {
         'units': {

bundles/proxmox-ve/items.py

@@ -0,0 +1,21 @@
+files = {
+    '/etc/apt/apt.conf.d/10pveapthook': {
+        'content_type': 'any',
+        'mode': '0644',
+    },
+    '/etc/apt/apt.conf.d/76pveconf': {
+        'content_type': 'any',
+        'mode': '0444',
+    },
+    '/etc/apt/apt.conf.d/76pveproxy': {
+        'content_type': 'any',
+        'mode': '0444',
+    },
+    '/etc/network/interfaces': {
+        'content_type': 'any',
+    },
+}
+
+symlinks['/etc/ssh/ssh_host_rsa_key.pub'] = {
+    'target': '/etc/ssh/ssh_host_managed_key.pub',
+}

bundles/proxmox-ve/metadata.py

@@ -0,0 +1,98 @@
+defaults = {
+    'apt': {
+        'packages': {
+            'linux-image-amd64': {
+                'installed': False,
+            },
+            'proxmox-default-kernel': {},
+            # after reboot
+            'proxmox-ve': {},
+            'postfix': {},
+            'open-iscsi': {},
+            'chrony': {},
+            'os-prober': {
+                'installed': False,
+            },
+        },
+        'sources': {
+            'proxmox-ve': {
+                'options': {
+                    'aarch': 'amd64',
+                },
+                'urls': {
+                    'http://download.proxmox.com/debian/pve',
+                },
+                'suites': {
+                    '{codename}',
+                },
+                'components': {
+                    'pve-no-subscription',
+                },
+                'key': 'proxmox-ve-{codename}',
+            },
+        },
+    },
+    # 'nftables': {
+    #     'input': {
+    #         'tcp dport 8006 accept',
+    #     },
+    # },
+    'zfs': {
+        'datasets': {
+            'tank/proxmox-ve': {
+                'mountpoint': '/var/lib/proxmox-ve',
+            },
+        }
+    }
+}
+
+
+# @metadata_reactor.provides(
+#     'systemd',
+# )
+# def bridge(metadata):
+#     return {
+#         'systemd': {
+#             'units': {
+#                 # f'internal.network': {
+#                 #     'Network': {
+#                 #         'Bridge': 'br0',
+#                 #     },
+#                 # },
+#                 'br0.netdev': {
+#                     'NetDev': {
+#                         'Name': 'br0',
+#                         'Kind': 'bridge'
+#                     },
+#                 },
+#                 'br0.network': {
+#                     'Match': {
+#                         'Name': 'br0',
+#                     },
+#                     'Network': {
+#                         'Unmanaged': 'yes'
+#                     },
+#                 },
+#             },
+#         },
+#     }
+
+
+@metadata_reactor.provides(
+    'nginx/vhosts',
+)
+def nginx(metadata):
+    return {
+        'nginx': {
+            'has_websockets': True,
+            'vhosts': {
+                metadata.get('proxmox-ve/domain'): {
+                    'content': 'nginx/proxy_pass.conf',
+                    'context': {
+                        'target': 'https://localhost:8006',
+                        'websockets': True,
+                    }
+                },
+            },
+        },
+    }

bundles/redis/items.py

@@ -2,12 +2,14 @@ directories = {
     '/etc/redis': {
         'purge': True,
         'owner': 'redis',
+        'mode': '2770',
         'needs': [
             'pkg_apt:redis-server',
         ],
     },
     '/var/lib/redis': {
         'owner': 'redis',
+        'mode': '0750',
         'needs': [
             'pkg_apt:redis-server',
         ],

bundles/systemd-networkd/files/resolv.conf

@@ -1,3 +1,10 @@
-% for nameserver in sorted(node.metadata.get('nameservers')):
+<%
+    nameservers = (
+        node.metadata.get('overwrite_nameservers', []) or
+        node.metadata.get('nameservers', [])
+    )
+%>\
+\
+% for nameserver in nameservers:
 nameserver ${nameserver}
 % endfor

bundles/systemd-networkd/items.py

@@ -1,9 +1,6 @@
 assert node.has_bundle('systemd')
 
 files = {
-    '/etc/network/interfaces': {
-        'delete': True,
-    },
     '/etc/resolv.conf': {
         'content_type': 'mako',
     },
@@ -19,5 +16,11 @@ directories = {
 }
 
 svc_systemd = {
-    'systemd-networkd': {},
+    'systemd-networkd.service': {},
+}
+
+
+if not node.has_bundle('proxmox-ve'):
+    files['/etc/network/interfaces'] = {
+        'delete': True,
     }

bundles/systemd/items.py

@@ -24,10 +24,10 @@ for name, unit in node.metadata.get('systemd/units').items():
         path = f'/etc/systemd/network/{name}'
         dependencies = {
             'needed_by': [
-                'svc_systemd:systemd-networkd',
+                'svc_systemd:systemd-networkd.service',
             ],
             'triggers': [
-                'svc_systemd:systemd-networkd:restart',
+                'svc_systemd:systemd-networkd.service:restart',
             ],
         }
     elif extension in ['timer', 'service', 'mount', 'swap', 'target']:

bundles/wireguard/metadata.py

@@ -12,7 +12,7 @@ defaults = {
             'wireguard': {
                 'backports': node.os_version < (11,),
                 'triggers': [
-                    'svc_systemd:systemd-networkd:restart',
+                    'svc_systemd:systemd-networkd.service:restart',
                 ],
             },
         },

bundles/yourls/files/config.php

@@ -0,0 +1,24 @@
+<?php
+define( 'YOURLS_DB_USER', 'yourls' );
+define( 'YOURLS_DB_PASS', '${db_password}' );
+define( 'YOURLS_DB_NAME', 'yourls' );
+define( 'YOURLS_DB_HOST', 'localhost' );
+define( 'YOURLS_DB_PREFIX', 'yourls_' );
+
+define( 'YOURLS_SITE', 'https://${hostname}' );
+define( 'YOURLS_LANG', '' );
+define( 'YOURLS_UNIQUE_URLS', true );
+define( 'YOURLS_PRIVATE', true );
+define( 'YOURLS_COOKIEKEY', '${cookiekey}' );
+
+$yourls_user_passwords = [
+% for username, password in users.items():
+	'${username}' => '${password}',
+% endfor
+];
+
+define( 'YOURLS_URL_CONVERT', 36 );
+
+define( 'YOURLS_DEBUG', false );
+
+$yourls_reserved_URL = [];

bundles/yourls/items.py

@@ -0,0 +1,48 @@
+directories = {
+    '/var/www/yourls/htdocs': {
+        'owner': 'www-data',
+        'group': 'www-data',
+        'mode': '0755',
+    },
+}
+
+git_deploy = {
+    '/var/www/yourls/htdocs': {
+        'repo': 'https://github.com/YOURLS/YOURLS.git',
+        'rev': node.metadata.get('yourls/version'),
+        'needs': [
+            'directory:/var/www/yourls/htdocs',
+        ],
+        'triggers': [
+            'svc_systemd:nginx:restart',
+        ],
+    },
+}
+
+files = {
+    f'/var/www/yourls/htdocs/user/config.php': {
+        'content_type': 'mako',
+        'mode': '0440',
+        'owner': 'www-data',
+        'group': 'www-data',
+        'context': {
+            'db_password': node.metadata.get('mariadb/databases/yourls/password'),
+            'hostname': node.metadata.get('yourls/hostname'),
+            'cookiekey': node.metadata.get('yourls/cookiekey'),
+            'users': node.metadata.get('yourls/users'),
+        },
+        'needs': [
+            'git_deploy:/var/www/yourls/htdocs',
+        ],
+        'triggers': [
+            'svc_systemd:nginx:restart',
+        ],
+    },
+
+    # FIXME:
+    '/var/www/certbot': {
+        'owner': 'www-data',
+        'group': 'www-data',
+        'mode': '0755',
+    }
+}

bundles/yourls/metadata.py

@@ -0,0 +1,42 @@
+defaults = {
+    'mariadb': {
+        'databases': {
+            'yourls': {
+                'password': repo.vault.random_bytes_as_base64_for(f'{node.name} yourls DB', length=32).value,
+            },
+        },
+    },
+}
+
+
+@metadata_reactor.provides(
+    'apt/packages',
+)
+def apt(metadata):
+    php_version = metadata.get('php/version')
+
+    return {
+        'apt':{
+            'packages': {
+                f'php{php_version}-mysql': {},
+            },
+        },
+    }
+
+
+@metadata_reactor.provides(
+    'nginx/vhosts',
+)
+def nginx(metadata):
+    return {
+        'nginx': {
+            'vhosts': {
+                metadata.get('yourls/hostname'): {
+                    'content': 'yourls/vhost.conf',
+                    'context': {
+                        'php_version': metadata.get('php/version'),
+                    },
+                },
+            },
+        },
+    }

data/grafana/vhost.conf

@@ -1,8 +1,3 @@
-map $http_upgrade $connection_upgrade {
-  default upgrade;
-  '' close;
-}
-
 server {
   listen 443 ssl http2;
   listen [::]:443 ssl http2;

data/nginx/proxy_pass.conf

@@ -8,6 +8,10 @@ server {
 
     location / {
         proxy_set_header   X-Real-IP          $remote_addr;
+% if websockets:
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection $connection_upgrade;
+% endif
         proxy_pass ${target};
     }
 }

data/yourls/vhost.conf

@@ -0,0 +1,31 @@
+server {
+  listen 443 ssl http2;
+  listen [::]:443 ssl http2;
+  server_name ${server_name};
+
+  ssl_certificate /etc/letsencrypt/archive/${server_name}/fullchain1.pem;
+  ssl_certificate_key /etc/letsencrypt/archive/${server_name}/privkey1.pem;
+
+  root /var/www/yourls/htdocs;
+
+  location / {
+    index index.php index.html index.htm;
+    try_files $uri $uri/ /yourls-loader.php$is_args$args;
+  }
+
+  location ~ \.php$ {
+    include params/fastcgi;
+    fastcgi_index index.php;
+    fastcgi_pass unix:/run/php/php${php_version}-fpm.sock;
+  }
+
+  # temp
+  location ^~ /.well-known/acme-challenge/ {
+      alias /var/www/certbot/;
+  }
+}
+
+
+# FIXME: this is a temporary solution to allow the certbot challenge to work:
+# - ssl_certificate
+# - ssl_certificate_key

groups.py

@@ -6,4 +6,8 @@ for root, dirs, files in walk(join(repo_path, "groups")):
         if filename.endswith(".py"):
             group = join(root, filename)
             with open(group, 'r', encoding='utf-8') as f:
+                try:
                     groups[splitext(basename(filename))[0]] = eval(f.read())
+                except:
+                    print(f"Error parsing {group}:")
+                    raise

groups/os/debian-11.py

@@ -2,6 +2,9 @@
     'supergroups': [
         'debian',
     ],
+    'bundles': [
+        'systemd-networkd',
+    ],
     'metadata': {
         'php': {
             'version': '7.4',

groups/os/debian-12-common.py

@@ -0,0 +1,26 @@
+{
+    'metadata': {
+        'apt': {
+            'sources': {
+                'debian': {
+                    'components': {
+                        'non-free-firmware',
+                    },
+                },
+                'debian-security': {
+                    'components': {
+                        'non-free-firmware',
+                    },
+                },
+            },
+        },
+        'php': {
+            'version': '8.2',
+        },
+        'postgresql': {
+            'version': '15',
+        },
+        'os_codename': 'bookworm',
+    },
+    'os_version': (12,),
+}

groups/os/debian-12-pve.py

@@ -0,0 +1,9 @@
+{
+    'supergroups': [
+        'debian',
+        'debian-12-common',
+    ],
+    'bundles': [
+        'ifupdown',
+    ],
+}

groups/os/debian-12.py

@@ -1,6 +1,10 @@
 {
     'supergroups': [
         'debian',
+        'debian-12-common',
+    ],
+    'bundles': [
+        'systemd-networkd',
     ],
     'metadata': {
         'apt': {

groups/os/linux.py

@@ -14,7 +14,6 @@
         'system',
         'systemd',
         'systemd-journald',
-        'systemd-networkd',
         'systemd-mount',
         'systemd-timers',
         'users',

libs/rsa.py

@@ -1,12 +1,10 @@
 # https://stackoverflow.com/a/18266970
 
 from Crypto.PublicKey import RSA
-from Crypto.Hash import HMAC
 from struct import pack
 from hashlib import sha3_512
 from cryptography.hazmat.primitives.serialization import load_der_private_key
 from functools import cache
-from cache_to_disk import cache_to_disk
 
 
 class PRNG(object):
@@ -22,7 +20,6 @@ class PRNG(object):
         return result
 
 
-@cache_to_disk(30)
 def _generate_deterministic_rsa_private_key(secret_bytes):
     return RSA.generate(2048, randfunc=PRNG(secret_bytes)).export_key('DER')
 

nodes/home.router.py

@@ -18,7 +18,7 @@
                 'interface': 'enx00e04c220682',
                 'ipv4': '10.0.99.126/24',
                 'gateway4': '10.0.99.1',
-                'vlans': {'iot', 'internet', 'guest', 'rolf', 'internal'},
+                'vlans': {'iot', 'internet', 'guest', 'rolf', 'internal', 'proxmox'},
             },
             'internal': {
                 'type': 'vlan',
@@ -37,6 +37,12 @@
                 'id': 3,
                 'ipv4': '10.0.3.1/24',
             },
+            'proxmox': {
+                'type': 'vlan',
+                'id': 4,
+                'ipv4': '10.0.4.1/24',
+                'dhcp_server': True,
+            },
             'guest': {
                 'type': 'vlan',
                 'id': 9,

nodes/home.server.py

@@ -35,6 +35,7 @@
         #'tasmota-charge',
         'wol-waker',
         'zfs',
+        'proxmox-ve',
     ],
     'metadata': {
         'id': 'af96709e-b13f-4965-a588-ef2cd476437a',
@@ -47,7 +48,7 @@
         },
         'apt': {
             'packages': {
-                'firmware-realtek': {},
+                # 'firmware-realtek': {}, proxmox-ve incompatibility
             },
         },
         'build-server': {
@@ -124,6 +125,9 @@
                 'unsortable': 'SofortUpload/Unsortable',
             },
         },
+        'proxmox-ve': {
+            'domain': 'pve.ckn.li',
+        },
         'raspberrymatic-cert': {
             'domain': 'homematic.ckn.li',
             'node': 'home.homematic',

nodes/mseibert.yourls.py

@@ -0,0 +1,60 @@
+# https://teamvault.apps.seibert-media.net/secrets/mkqMRv/
+# https://console.hetzner.cloud/projects/889138/servers/46578341
+
+{
+    'hostname': '168.119.250.114',
+    'groups': [
+        #'backup',
+        'debian-12',
+        #'monitored',
+        'webserver',
+    ],
+    'bundles': [
+        #'wireguard',
+        'mariadb',
+        'php',
+        'yourls',
+        'zfs',
+    ],
+    'metadata': {
+        'id': '52efcd47-edd8-426c-aead-c492553d14f9',
+        'network': {
+            'internal': {
+                'interface': 'ens10',
+                'ipv4': '10.0.227.4/24',
+            },
+            'external': {
+                'interface': 'eth0',
+                'ipv4': '168.119.250.114/32',
+                'gateway4': '172.31.1.1',
+                'ipv6': '2a01:4f8:c013:e321::2/64',
+                'gateway6': 'fe80::1',
+            },
+        },
+        'yourls': {
+            'hostname': "direkt.oranienschule.de",
+            'cookiekey': "!decrypt:encrypt$gAAAAABoRvmcUs3t7PREllyeN--jBqs0XYewMHW16GWC-ikLzsDSe02YKGycOlgXuHU4hzKbNjGMEutpFXRLk9Zji6bbpy4GdyE6vStfwd8ZT0obAyoqBPwI47LwUlDSFMS51y5j8rG5",
+            'version': "1.10.1",
+            'users': {
+                'mseibert': "!decrypt:encrypt$gAAAAABoRwtOcslyRY9ahkmtVI8QbXgJhyE3nuk04eakFDKl-4OZViiRvjtQW3Uwqki1aFeAS-syzr0Ug5sZM_zNelNahjZyzW1k47Xg9GltGNn_zp-uUII=",
+            },
+        },
+        # FIXME:
+        'overwrite_nameservers': [
+            '8.8.8.8',
+        ],
+        'vm': {
+            'cores': 2,
+            'ram': 4096,
+        },
+        'zfs': {
+            'pools': {
+                'tank': {
+                    'devices': [
+                        '/var/lib/zfs_file',
+                    ],
+                },
+            },
+        },
+    },
+}

nodes/wb.offsite-backups.py

@@ -1,4 +1,5 @@
 {
+    'dummy': True,
     'hostname': '192.168.179.20',
     'groups': [
         'debian-12',

requirements.txt

@@ -3,5 +3,4 @@ pycryptodome
 PyNaCl
 PyYAML
 pyqrcode
-cache_to_disk
 setuptools