From 35243fdba6032192ece3a3177ad152d2837945d9 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sun, 22 Jun 2025 09:35:35 +0200 Subject: [PATCH 01/13] offsitebackup offline --- nodes/wb.offsite-backups.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nodes/wb.offsite-backups.py b/nodes/wb.offsite-backups.py index f72c017..0fa1017 100644 --- a/nodes/wb.offsite-backups.py +++ b/nodes/wb.offsite-backups.py @@ -1,4 +1,5 @@ { + 'dummy': True, 'hostname': '192.168.179.20', 'groups': [ 'debian-12', From 1d8361cc5fa035ac4e02c1f4c670c58044f369e4 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sun, 22 Jun 2025 09:36:21 +0200 Subject: [PATCH 02/13] cache_to_disk broken --- libs/rsa.py | 3 --- requirements.txt | 1 - 2 files changed, 4 deletions(-) diff --git a/libs/rsa.py b/libs/rsa.py index e2666fb..60caabd 100644 --- a/libs/rsa.py +++ b/libs/rsa.py @@ -1,12 +1,10 @@ # https://stackoverflow.com/a/18266970 from Crypto.PublicKey import RSA -from Crypto.Hash import HMAC from struct import pack from hashlib import sha3_512 from cryptography.hazmat.primitives.serialization import load_der_private_key from functools import cache -from cache_to_disk import cache_to_disk class PRNG(object): @@ -22,7 +20,6 @@ class PRNG(object): return result -@cache_to_disk(30) def _generate_deterministic_rsa_private_key(secret_bytes): return RSA.generate(2048, randfunc=PRNG(secret_bytes)).export_key('DER') diff --git a/requirements.txt b/requirements.txt index 0f80ca6..75cb897 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,4 @@ pycryptodome PyNaCl PyYAML pyqrcode -cache_to_disk setuptools From dc40295ddea1fd59cbc5462bbab2f6df9e80aa90 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sun, 22 Jun 2025 09:36:56 +0200 Subject: [PATCH 03/13] print message on parsing group error --- groups.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/groups.py b/groups.py index 6674fe8..05999a8 100644 --- a/groups.py +++ b/groups.py @@ -6,4 +6,8 @@ for root, dirs, files in walk(join(repo_path, "groups")): if filename.endswith(".py"): group = join(root, filename) with open(group, 'r', encoding='utf-8') as f: - groups[splitext(basename(filename))[0]] = eval(f.read()) + try: + groups[splitext(basename(filename))[0]] = eval(f.read()) + except: + print(f"Error parsing {group}:") + raise From 6616ae7417f6573229273ab69bd2cc5bd18630bd Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sun, 22 Jun 2025 09:37:16 +0200 Subject: [PATCH 04/13] fix some redis permissions --- bundles/redis/items.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bundles/redis/items.py b/bundles/redis/items.py index 91e5821..12b8a6e 100644 --- a/bundles/redis/items.py +++ b/bundles/redis/items.py @@ -2,12 +2,14 @@ directories = { '/etc/redis': { 'purge': True, 'owner': 'redis', + 'mode': '2770', 'needs': [ 'pkg_apt:redis-server', ], }, '/var/lib/redis': { 'owner': 'redis', + 'mode': '0750', 'needs': [ 'pkg_apt:redis-server', ], From 3dffc05c9d798bd5ca8181265963c57b990fd7b6 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sun, 22 Jun 2025 09:40:13 +0200 Subject: [PATCH 05/13] apt add docs about options --- bundles/apt/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bundles/apt/README.md b/bundles/apt/README.md index 012d490..88d9c21 100644 --- a/bundles/apt/README.md +++ b/bundles/apt/README.md @@ -13,6 +13,9 @@ 'deb', 'deb-src', }, + 'options': { # optional + 'aarch': 'amd64', + }, 'urls': { 'https://deb.debian.org/debian', }, From d3ba9db0c6bd1666ddb995ec000c79809d811ff0 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sun, 22 Jun 2025 09:40:26 +0200 Subject: [PATCH 06/13] maybe keep etc/kernel/postinst.d/apt-auto-removal? --- bundles/apt/items.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bundles/apt/items.py b/bundles/apt/items.py index a8da430..853c5c8 100644 --- a/bundles/apt/items.py +++ b/bundles/apt/items.py @@ -62,6 +62,7 @@ files = { '/usr/lib/nagios/plugins/check_apt_upgradable': { 'mode': '0755', }, + # /etc/kernel/postinst.d/apt-auto-removal } actions = { From bdb9fa064d51b7d4051e8b70eaf3861f4479c5d1 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sun, 22 Jun 2025 09:40:41 +0200 Subject: [PATCH 07/13] gitea disable registration --- bundles/gitea/files/app.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundles/gitea/files/app.ini b/bundles/gitea/files/app.ini index 71539cc..56acb8d 100644 --- a/bundles/gitea/files/app.ini +++ b/bundles/gitea/files/app.ini @@ -40,7 +40,7 @@ ENABLE_OPENID_SIGNUP = false [service] REGISTER_EMAIL_CONFIRM = true ENABLE_NOTIFY_MAIL = true -DISABLE_REGISTRATION = false +DISABLE_REGISTRATION = true ALLOW_ONLY_EXTERNAL_REGISTRATION = false ENABLE_CAPTCHA = false REQUIRE_SIGNIN_VIEW = false From 187b0440c888d58f4873f634103ecebf00c91fa5 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sun, 22 Jun 2025 09:49:27 +0200 Subject: [PATCH 08/13] nginx use expected dirs and allow websockets in proxy pass --- bundles/grafana/metadata.py | 4 ++++ bundles/nginx/files/nginx.conf | 10 +++++++++- bundles/nginx/items.py | 20 +++++++++++++++++--- bundles/nginx/metadata.py | 1 + data/grafana/vhost.conf | 7 +------ data/nginx/proxy_pass.conf | 4 ++++ 6 files changed, 36 insertions(+), 10 deletions(-) diff --git a/bundles/grafana/metadata.py b/bundles/grafana/metadata.py index 4ac5259..5b68a2b 100644 --- a/bundles/grafana/metadata.py +++ b/bundles/grafana/metadata.py @@ -69,6 +69,9 @@ defaults = { }, }, }, + 'nginx': { + 'has_websockets': True, + }, } @@ -144,6 +147,7 @@ def dns(metadata): def nginx(metadata): return { 'nginx': { + 'has_websockets': True, 'vhosts': { metadata.get('grafana/hostname'): { 'content': 'grafana/vhost.conf', diff --git a/bundles/nginx/files/nginx.conf b/bundles/nginx/files/nginx.conf index f4beed0..ef45635 100644 --- a/bundles/nginx/files/nginx.conf +++ b/bundles/nginx/files/nginx.conf @@ -31,5 +31,13 @@ http { } % endif - include /etc/nginx/sites/*; + + % if has_websockets: + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + % endif + + include /etc/nginx/sites-enabled/*; } diff --git a/bundles/nginx/items.py b/bundles/nginx/items.py index b8f5849..5f80b74 100644 --- a/bundles/nginx/items.py +++ b/bundles/nginx/items.py @@ -9,7 +9,7 @@ directories = { 'svc_systemd:nginx:restart', }, }, - '/etc/nginx/sites': { + '/etc/nginx/sites-available': { 'purge': True, 'triggers': { 'svc_systemd:nginx:restart', @@ -25,6 +25,13 @@ directories = { 'purge': True, 'owner': 'www-data', }, + + # temp + '/var/www/certbot': { + 'owner': 'www-data', + 'group': 'www-data', + 'mode': '0755', + } } files = { @@ -33,6 +40,7 @@ files = { 'context': { 'modules': node.metadata.get('nginx/modules'), 'worker_processes': node.metadata.get('vm/cores'), + 'has_websockets': node.metadata.get('nginx/has_websockets'), }, 'triggers': { 'svc_systemd:nginx:restart', @@ -75,6 +83,12 @@ files = { }, } +symlinks = { + '/etc/nginx/sites-enabled': { + 'target': '/etc/nginx/sites-available', + }, +} + actions = { 'nginx-generate-dhparam': { 'command': 'openssl dhparam -dsaparam -out /etc/ssl/certs/dhparam.pem 4096', @@ -93,7 +107,7 @@ svc_systemd = { for name, config in node.metadata.get('nginx/vhosts').items(): - files[f'/etc/nginx/sites/{name}'] = { + files[f'/etc/nginx/sites-available/{name}'] = { 'content': Template(filename=join(repo.path, 'data', config['content'])).render( server_name=name, **config.get('context', {}), @@ -109,6 +123,6 @@ for name, config in node.metadata.get('nginx/vhosts').items(): } if name in node.metadata.get('letsencrypt/domains'): - files[f'/etc/nginx/sites/{name}']['needs'].append( + files[f'/etc/nginx/sites-available/{name}']['needs'].append( f'action:letsencrypt_ensure-some-certificate_{name}', ) diff --git a/bundles/nginx/metadata.py b/bundles/nginx/metadata.py index a5ab350..37f3f8a 100644 --- a/bundles/nginx/metadata.py +++ b/bundles/nginx/metadata.py @@ -18,6 +18,7 @@ defaults = { 'nginx': { 'vhosts': {}, 'modules': set(), + 'has_websockets': False, }, 'systemd': { 'units': { diff --git a/data/grafana/vhost.conf b/data/grafana/vhost.conf index c8c395e..81ba4d6 100644 --- a/data/grafana/vhost.conf +++ b/data/grafana/vhost.conf @@ -1,13 +1,8 @@ -map $http_upgrade $connection_upgrade { - default upgrade; - '' close; -} - server { listen 443 ssl http2; listen [::]:443 ssl http2; server_name ${server_name}; - + ssl_certificate /var/lib/dehydrated/certs/${server_name}/fullchain.pem; ssl_certificate_key /var/lib/dehydrated/certs/${server_name}/privkey.pem; diff --git a/data/nginx/proxy_pass.conf b/data/nginx/proxy_pass.conf index 7d3069f..d682396 100644 --- a/data/nginx/proxy_pass.conf +++ b/data/nginx/proxy_pass.conf @@ -8,6 +8,10 @@ server { location / { proxy_set_header X-Real-IP $remote_addr; +% if websockets: + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; +% endif proxy_pass ${target}; } } From 663116c778e6658d5375382c3ee231fb32bc27e2 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sun, 22 Jun 2025 09:50:37 +0200 Subject: [PATCH 09/13] /var/lib/mysql needs mysql user to exist --- bundles/mariadb/items.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundles/mariadb/items.py b/bundles/mariadb/items.py index 07dffc3..58fd34e 100644 --- a/bundles/mariadb/items.py +++ b/bundles/mariadb/items.py @@ -11,7 +11,7 @@ directories = { 'needs': [ 'zfs_dataset:tank/mariadb', ], - 'needed_by': [ + 'needs': [ 'pkg_apt:mariadb-server', 'pkg_apt:mariadb-client', ], From befdf5ad6e71714779985d548c0a761b0b131231 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sun, 22 Jun 2025 09:51:50 +0200 Subject: [PATCH 10/13] fixmo mariadb dependency --- bundles/mariadb/metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bundles/mariadb/metadata.py b/bundles/mariadb/metadata.py index ee406cb..8b65e31 100644 --- a/bundles/mariadb/metadata.py +++ b/bundles/mariadb/metadata.py @@ -3,12 +3,12 @@ defaults = { 'packages': { 'mariadb-server': { 'needs': { - 'zfs_dataset:tank/mariadb', + #'zfs_dataset:tank/mariadb', }, }, 'mariadb-client': { 'needs': { - 'zfs_dataset:tank/mariadb', + #'zfs_dataset:tank/mariadb', }, }, }, From 9733a55942f5deaae97e976b004e71304e1133e5 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sun, 22 Jun 2025 09:53:22 +0200 Subject: [PATCH 11/13] svc_systemd:systemd-networkd add .service to name --- bundles/kea-dhcpd/items.py | 2 +- bundles/systemd-networkd/items.py | 3 ++- bundles/systemd/items.py | 4 ++-- bundles/wireguard/metadata.py | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/bundles/kea-dhcpd/items.py b/bundles/kea-dhcpd/items.py index ba5846d..5d1fd00 100644 --- a/bundles/kea-dhcpd/items.py +++ b/bundles/kea-dhcpd/items.py @@ -15,7 +15,7 @@ svc_systemd = { 'needs': [ 'pkg_apt:kea-dhcp4-server', 'file:/etc/kea/kea-dhcp4.conf', - 'svc_systemd:systemd-networkd:restart', + 'svc_systemd:systemd-networkd.service:restart', ], }, } diff --git a/bundles/systemd-networkd/items.py b/bundles/systemd-networkd/items.py index 07be82b..47bb2bd 100644 --- a/bundles/systemd-networkd/items.py +++ b/bundles/systemd-networkd/items.py @@ -19,5 +19,6 @@ directories = { } svc_systemd = { - 'systemd-networkd': {}, + 'systemd-networkd.service': {}, } + diff --git a/bundles/systemd/items.py b/bundles/systemd/items.py index 7927115..7816130 100644 --- a/bundles/systemd/items.py +++ b/bundles/systemd/items.py @@ -24,10 +24,10 @@ for name, unit in node.metadata.get('systemd/units').items(): path = f'/etc/systemd/network/{name}' dependencies = { 'needed_by': [ - 'svc_systemd:systemd-networkd', + 'svc_systemd:systemd-networkd.service', ], 'triggers': [ - 'svc_systemd:systemd-networkd:restart', + 'svc_systemd:systemd-networkd.service:restart', ], } elif extension in ['timer', 'service', 'mount', 'swap', 'target']: diff --git a/bundles/wireguard/metadata.py b/bundles/wireguard/metadata.py index b4d1c13..f06e259 100644 --- a/bundles/wireguard/metadata.py +++ b/bundles/wireguard/metadata.py @@ -12,7 +12,7 @@ defaults = { 'wireguard': { 'backports': node.os_version < (11,), 'triggers': [ - 'svc_systemd:systemd-networkd:restart', + 'svc_systemd:systemd-networkd.service:restart', ], }, }, From 9e139fd422c4001d9c9a95d53d1f0a3362fcb155 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sun, 22 Jun 2025 10:03:38 +0200 Subject: [PATCH 12/13] fix remove leftover --- bundles/nginx/items.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/bundles/nginx/items.py b/bundles/nginx/items.py index 5f80b74..7ce0fce 100644 --- a/bundles/nginx/items.py +++ b/bundles/nginx/items.py @@ -25,13 +25,6 @@ directories = { 'purge': True, 'owner': 'www-data', }, - - # temp - '/var/www/certbot': { - 'owner': 'www-data', - 'group': 'www-data', - 'mode': '0755', - } } files = { From aeb0a4fbe7b46ee85d39e240c479670eedae1c67 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sun, 22 Jun 2025 10:07:10 +0200 Subject: [PATCH 13/13] nodes/mseibert.yourls.py: introduce --- bundles/systemd-networkd/files/resolv.conf | 11 +++- bundles/yourls/files/config.php | 24 +++++++++ bundles/yourls/items.py | 48 +++++++++++++++++ bundles/yourls/metadata.py | 42 +++++++++++++++ data/yourls/vhost.conf | 31 +++++++++++ nodes/mseibert.yourls.py | 60 ++++++++++++++++++++++ 6 files changed, 214 insertions(+), 2 deletions(-) create mode 100644 bundles/yourls/files/config.php create mode 100644 bundles/yourls/items.py create mode 100644 bundles/yourls/metadata.py create mode 100644 data/yourls/vhost.conf create mode 100644 nodes/mseibert.yourls.py diff --git a/bundles/systemd-networkd/files/resolv.conf b/bundles/systemd-networkd/files/resolv.conf index e4f8999..36cabb2 100644 --- a/bundles/systemd-networkd/files/resolv.conf +++ b/bundles/systemd-networkd/files/resolv.conf @@ -1,3 +1,10 @@ -% for nameserver in sorted(node.metadata.get('nameservers')): +<% + nameservers = ( + node.metadata.get('overwrite_nameservers', []) or + node.metadata.get('nameservers', []) + ) +%>\ +\ +% for nameserver in nameservers: nameserver ${nameserver} -% endfor +% endfor \ No newline at end of file diff --git a/bundles/yourls/files/config.php b/bundles/yourls/files/config.php new file mode 100644 index 0000000..9be0a49 --- /dev/null +++ b/bundles/yourls/files/config.php @@ -0,0 +1,24 @@ + '${password}', +% endfor +]; + +define( 'YOURLS_URL_CONVERT', 36 ); + +define( 'YOURLS_DEBUG', false ); + +$yourls_reserved_URL = []; \ No newline at end of file diff --git a/bundles/yourls/items.py b/bundles/yourls/items.py new file mode 100644 index 0000000..d9eb886 --- /dev/null +++ b/bundles/yourls/items.py @@ -0,0 +1,48 @@ +directories = { + '/var/www/yourls/htdocs': { + 'owner': 'www-data', + 'group': 'www-data', + 'mode': '0755', + }, +} + +git_deploy = { + '/var/www/yourls/htdocs': { + 'repo': 'https://github.com/YOURLS/YOURLS.git', + 'rev': node.metadata.get('yourls/version'), + 'needs': [ + 'directory:/var/www/yourls/htdocs', + ], + 'triggers': [ + 'svc_systemd:nginx:restart', + ], + }, +} + +files = { + f'/var/www/yourls/htdocs/user/config.php': { + 'content_type': 'mako', + 'mode': '0440', + 'owner': 'www-data', + 'group': 'www-data', + 'context': { + 'db_password': node.metadata.get('mariadb/databases/yourls/password'), + 'hostname': node.metadata.get('yourls/hostname'), + 'cookiekey': node.metadata.get('yourls/cookiekey'), + 'users': node.metadata.get('yourls/users'), + }, + 'needs': [ + 'git_deploy:/var/www/yourls/htdocs', + ], + 'triggers': [ + 'svc_systemd:nginx:restart', + ], + }, + + # FIXME: + '/var/www/certbot': { + 'owner': 'www-data', + 'group': 'www-data', + 'mode': '0755', + } +} diff --git a/bundles/yourls/metadata.py b/bundles/yourls/metadata.py new file mode 100644 index 0000000..ed09937 --- /dev/null +++ b/bundles/yourls/metadata.py @@ -0,0 +1,42 @@ +defaults = { + 'mariadb': { + 'databases': { + 'yourls': { + 'password': repo.vault.random_bytes_as_base64_for(f'{node.name} yourls DB', length=32).value, + }, + }, + }, +} + + +@metadata_reactor.provides( + 'apt/packages', +) +def apt(metadata): + php_version = metadata.get('php/version') + + return { + 'apt':{ + 'packages': { + f'php{php_version}-mysql': {}, + }, + }, + } + + +@metadata_reactor.provides( + 'nginx/vhosts', +) +def nginx(metadata): + return { + 'nginx': { + 'vhosts': { + metadata.get('yourls/hostname'): { + 'content': 'yourls/vhost.conf', + 'context': { + 'php_version': metadata.get('php/version'), + }, + }, + }, + }, + } diff --git a/data/yourls/vhost.conf b/data/yourls/vhost.conf new file mode 100644 index 0000000..d1431df --- /dev/null +++ b/data/yourls/vhost.conf @@ -0,0 +1,31 @@ +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name ${server_name}; + + ssl_certificate /etc/letsencrypt/archive/${server_name}/fullchain1.pem; + ssl_certificate_key /etc/letsencrypt/archive/${server_name}/privkey1.pem; + + root /var/www/yourls/htdocs; + + location / { + index index.php index.html index.htm; + try_files $uri $uri/ /yourls-loader.php$is_args$args; + } + + location ~ \.php$ { + include params/fastcgi; + fastcgi_index index.php; + fastcgi_pass unix:/run/php/php${php_version}-fpm.sock; + } + + # temp + location ^~ /.well-known/acme-challenge/ { + alias /var/www/certbot/; + } +} + + +# FIXME: this is a temporary solution to allow the certbot challenge to work: +# - ssl_certificate +# - ssl_certificate_key diff --git a/nodes/mseibert.yourls.py b/nodes/mseibert.yourls.py new file mode 100644 index 0000000..d0d08c3 --- /dev/null +++ b/nodes/mseibert.yourls.py @@ -0,0 +1,60 @@ +# https://teamvault.apps.seibert-media.net/secrets/mkqMRv/ +# https://console.hetzner.cloud/projects/889138/servers/46578341 + +{ + 'hostname': '168.119.250.114', + 'groups': [ + #'backup', + 'debian-12', + #'monitored', + 'webserver', + ], + 'bundles': [ + #'wireguard', + 'mariadb', + 'php', + 'yourls', + 'zfs', + ], + 'metadata': { + 'id': '52efcd47-edd8-426c-aead-c492553d14f9', + 'network': { + 'internal': { + 'interface': 'ens10', + 'ipv4': '10.0.227.4/24', + }, + 'external': { + 'interface': 'eth0', + 'ipv4': '168.119.250.114/32', + 'gateway4': '172.31.1.1', + 'ipv6': '2a01:4f8:c013:e321::2/64', + 'gateway6': 'fe80::1', + }, + }, + 'yourls': { + 'hostname': "direkt.oranienschule.de", + 'cookiekey': "!decrypt:encrypt$gAAAAABoRvmcUs3t7PREllyeN--jBqs0XYewMHW16GWC-ikLzsDSe02YKGycOlgXuHU4hzKbNjGMEutpFXRLk9Zji6bbpy4GdyE6vStfwd8ZT0obAyoqBPwI47LwUlDSFMS51y5j8rG5", + 'version': "1.10.1", + 'users': { + 'mseibert': "!decrypt:encrypt$gAAAAABoRwtOcslyRY9ahkmtVI8QbXgJhyE3nuk04eakFDKl-4OZViiRvjtQW3Uwqki1aFeAS-syzr0Ug5sZM_zNelNahjZyzW1k47Xg9GltGNn_zp-uUII=", + }, + }, + # FIXME: + 'overwrite_nameservers': [ + '8.8.8.8', + ], + 'vm': { + 'cores': 2, + 'ram': 4096, + }, + 'zfs': { + 'pools': { + 'tank': { + 'devices': [ + '/var/lib/zfs_file', + ], + }, + }, + }, + }, +}