From d755267dd9611d13fd6f6791c28b1325ef4c67e1 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sun, 22 Jun 2025 22:22:52 +0200 Subject: [PATCH 01/15] proxmox --- bundles/ifupdown/items.py | 3 + bundles/network/metadata.py | 103 +++++++++++++------------- bundles/proxmox-ve/items.py | 21 ++++++ bundles/proxmox-ve/metadata.py | 99 +++++++++++++++++++++++++ bundles/systemd-networkd/items.py | 1 - data/apt/keys/proxmox-ve-bookworm.gpg | Bin 0 -> 1187 bytes groups/os/debian-11.py | 3 + groups/os/debian-12-common.py | 26 +++++++ groups/os/debian-12-pve.py | 10 +++ groups/os/debian-12.py | 4 + groups/os/linux.py | 1 - nodes/home.router.py | 8 +- nodes/home.server.py | 20 ++--- 13 files changed, 231 insertions(+), 68 deletions(-) create mode 100644 bundles/ifupdown/items.py create mode 100644 bundles/proxmox-ve/items.py create mode 100644 bundles/proxmox-ve/metadata.py create mode 100644 data/apt/keys/proxmox-ve-bookworm.gpg create mode 100644 groups/os/debian-12-common.py create mode 100644 groups/os/debian-12-pve.py diff --git a/bundles/ifupdown/items.py b/bundles/ifupdown/items.py new file mode 100644 index 0000000..9972a7e --- /dev/null +++ b/bundles/ifupdown/items.py @@ -0,0 +1,3 @@ +# svc_systemd = { +# 'ifupdown.service': {}, +# } diff --git a/bundles/network/metadata.py b/bundles/network/metadata.py index 8e8adbd..9e70910 100644 --- a/bundles/network/metadata.py +++ b/bundles/network/metadata.py @@ -36,61 +36,64 @@ def dhcp(metadata): 'systemd/units', ) def units(metadata): - units = {} + if node.has_bundle('systemd-networkd'): + units = {} - for network_name, network_conf in metadata.get('network').items(): - interface_type = network_conf.get('type', None) + for network_name, network_conf in metadata.get('network').items(): + interface_type = network_conf.get('type', None) - # network + # network - units[f'{network_name}.network'] = { - 'Match': { - 'Name': network_name if interface_type == 'vlan' else network_conf['interface'], - }, - 'Network': { - 'DHCP': network_conf.get('dhcp', 'no'), - 'IPv6AcceptRA': network_conf.get('dhcp', 'no'), - 'VLAN': set(network_conf.get('vlans', set())) - } - } - - # type - - if interface_type: - units[f'{network_name}.network']['Match']['Type'] = interface_type - - # ips - - for i in [4, 6]: - if network_conf.get(f'ipv{i}', None): - units[f'{network_name}.network'].update({ - f'Address#ipv{i}': { - 'Address': network_conf[f'ipv{i}'], - }, - }) - if f'gateway{i}' in network_conf: - units[f'{network_name}.network'].update({ - f'Route#ipv{i}': { - 'Gateway': network_conf[f'gateway{i}'], - 'GatewayOnlink': 'yes', - } - }) - - # as vlan - - if interface_type == 'vlan': - units[f"{network_name}.netdev"] = { - 'NetDev': { - 'Name': network_name, - 'Kind': 'vlan', + units[f'{network_name}.network'] = { + 'Match': { + 'Name': network_name if interface_type == 'vlan' else network_conf['interface'], }, - 'VLAN': { - 'Id': network_conf['id'], + 'Network': { + 'DHCP': network_conf.get('dhcp', 'no'), + 'IPv6AcceptRA': network_conf.get('dhcp', 'no'), + 'VLAN': set(network_conf.get('vlans', set())) } } - return { - 'systemd': { - 'units': units, + # type + + if interface_type: + units[f'{network_name}.network']['Match']['Type'] = interface_type + + # ips + + for i in [4, 6]: + if network_conf.get(f'ipv{i}', None): + units[f'{network_name}.network'].update({ + f'Address#ipv{i}': { + 'Address': network_conf[f'ipv{i}'], + }, + }) + if f'gateway{i}' in network_conf: + units[f'{network_name}.network'].update({ + f'Route#ipv{i}': { + 'Gateway': network_conf[f'gateway{i}'], + 'GatewayOnlink': 'yes', + } + }) + + # as vlan + + if interface_type == 'vlan': + units[f"{network_name}.netdev"] = { + 'NetDev': { + 'Name': network_name, + 'Kind': 'vlan', + }, + 'VLAN': { + 'Id': network_conf['id'], + } + } + + return { + 'systemd': { + 'units': units, + } } - } + else: + return {} diff --git a/bundles/proxmox-ve/items.py b/bundles/proxmox-ve/items.py new file mode 100644 index 0000000..d00540f --- /dev/null +++ b/bundles/proxmox-ve/items.py @@ -0,0 +1,21 @@ +files = { + '/etc/apt/apt.conf.d/10pveapthook': { + 'content_type': 'any', + 'mode': '0644', + }, + '/etc/apt/apt.conf.d/76pveconf': { + 'content_type': 'any', + 'mode': '0444', + }, + '/etc/apt/apt.conf.d/76pveproxy': { + 'content_type': 'any', + 'mode': '0444', + }, + '/etc/network/interfaces': { + 'content_type': 'any', + }, +} + +symlinks['/etc/ssh/ssh_host_rsa_key.pub'] = { + 'target': '/etc/ssh/ssh_host_managed_key.pub', +} diff --git a/bundles/proxmox-ve/metadata.py b/bundles/proxmox-ve/metadata.py new file mode 100644 index 0000000..0abe2ae --- /dev/null +++ b/bundles/proxmox-ve/metadata.py @@ -0,0 +1,99 @@ +defaults = { + 'apt': { + 'packages': { + 'linux-image-amd64': { + 'installed': False, + }, + 'proxmox-default-kernel': {}, + # after reboot + 'proxmox-ve': {}, + 'postfix': {}, + 'open-iscsi': {}, + 'chrony': {}, + 'os-prober': { + 'installed': False, + }, + 'dnsmasq-base': {}, + }, + 'sources': { + 'proxmox-ve': { + 'options': { + 'aarch': 'amd64', + }, + 'urls': { + 'http://download.proxmox.com/debian/pve', + }, + 'suites': { + '{codename}', + }, + 'components': { + 'pve-no-subscription', + }, + 'key': 'proxmox-ve-{codename}', + }, + }, + }, + # 'nftables': { + # 'input': { + # 'tcp dport 8006 accept', + # }, + # }, + 'zfs': { + 'datasets': { + 'tank/proxmox-ve': { + 'mountpoint': '/var/lib/proxmox-ve', + }, + } + } +} + + +# @metadata_reactor.provides( +# 'systemd', +# ) +# def bridge(metadata): +# return { +# 'systemd': { +# 'units': { +# # f'internal.network': { +# # 'Network': { +# # 'Bridge': 'br0', +# # }, +# # }, +# 'br0.netdev': { +# 'NetDev': { +# 'Name': 'br0', +# 'Kind': 'bridge' +# }, +# }, +# 'br0.network': { +# 'Match': { +# 'Name': 'br0', +# }, +# 'Network': { +# 'Unmanaged': 'yes' +# }, +# }, +# }, +# }, +# } + + +@metadata_reactor.provides( + 'nginx/vhosts', +) +def nginx(metadata): + return { + 'nginx': { + 'has_websockets': True, + 'vhosts': { + metadata.get('proxmox-ve/domain'): { + 'content': 'nginx/proxy_pass.conf', + 'context': { + 'target': 'https://localhost:8006', + 'websockets': True, + } + }, + }, + }, + } diff --git a/bundles/systemd-networkd/items.py b/bundles/systemd-networkd/items.py index 47bb2bd..af033ba 100644 --- a/bundles/systemd-networkd/items.py +++ b/bundles/systemd-networkd/items.py @@ -21,4 +21,3 @@ directories = { svc_systemd = { 'systemd-networkd.service': {}, } - diff --git a/data/apt/keys/proxmox-ve-bookworm.gpg b/data/apt/keys/proxmox-ve-bookworm.gpg new file mode 100644 index 0000000000000000000000000000000000000000..8e6e95e806e13e71e6175e9433e61c8b965edaec GIT binary patch literal 1187 zcmV;U1YG->0u2OXgJqln5CF^rXIo4a$3{!qI0{XK^pS@vp;huGOd+dKK}<2F8PfX1 zWf%Lxk-IT9WvvaMILjf4;grdsH+|(D6h18Mz72WuTXMP*9R(1aO)0SdrPuZ6k(zql`5Jf49aHba}!$=dmx-1s&1qbX!%19!rr1?HF zYsk1fmeD`5yL(+S(YnJgJa=oubCegXLn7|juqzqZBrgAuWo>Djgl>tqaPIC_y0>1@ z6|duk)NVxb_w7~8E_>YK6c@(KkbwPnl0uiwEX~3NN^qQW13nmd0KksaCa!iT!4w*W z_^5BYCr+zzb^$e{@NY?!ztqLujAb^Rfn2$xqg;}*+5acDnC)FT6No^;-=l@^!nPCB zqJFgQs#u50RcjZQ&ve$Y_ucVvkJSUv(P>I`TQoB42EFF}xxM+F%^5_VQ6?Cvmjgw~ zg-bqnEicROZd(Bo_djVbPI?u=`!uK0Px-|U2Bu9_u7IrJm;lj9G98GXAzRR&H@F2S z5$vJas!OX&3ZOvko-THjD0!scq6Y^aCQ<~7Gt2VA z?5BvxsA9&kMlDj=arG;cicsk#jpLH=)O$^-wyWp@MEcpDvu#p>Bjov+dvi%-+ ztJ284{ZW4X9!R19ds_5hHRQ>eYpav206~0TF+PygJ*R0sbbvx;l!m>{fU*JVH5FJ& z0l0c`O93|wL7+DbsZOW>uSfUNYW|2Lo!mDtf=FK={1#uKfkj~_@dYDOvpbC`y%QTt z8!xN|h3pm1A@Gfm^@`tn&S*uF2MH3ab6Px3>~9Di*gktsNs?4nHGZSR^G!I`gedM~ zSZXf4p8FQi zs<`~{1|zS;dAq<(lTB2_R*FA|s-x2M>&3@QO*RnzO#$ z!+~v^sL%u8W47sY1j5tedb;L>Jd^-?xipbWa2%u`Tb9HWE!?)_?y^#_ Date: Fri, 27 Jun 2025 15:02:11 +0200 Subject: [PATCH 02/15] mariadb use ini parser --- bundles/mariadb/files/override.conf | 11 ----------- bundles/mariadb/items.py | 6 ++---- libs/ini.py | 5 ++--- 3 files changed, 4 insertions(+), 18 deletions(-) delete mode 100644 bundles/mariadb/files/override.conf diff --git a/bundles/mariadb/files/override.conf b/bundles/mariadb/files/override.conf deleted file mode 100644 index ae2fccd..0000000 --- a/bundles/mariadb/files/override.conf +++ /dev/null @@ -1,11 +0,0 @@ -% for section, options in sorted(conf.items()): -[${section}] -% for key, value in sorted(options.items()): -% if value is None: -${key} -% else: -${key} = ${value} -% endif -% endfor - -% endfor diff --git a/bundles/mariadb/items.py b/bundles/mariadb/items.py index 58fd34e..7ff4e83 100644 --- a/bundles/mariadb/items.py +++ b/bundles/mariadb/items.py @@ -20,10 +20,8 @@ directories = { files = { '/etc/mysql/conf.d/override.conf': { - 'context': { - 'conf': node.metadata.get('mariadb/conf'), - }, - 'content_type': 'mako', + 'content': repo.libs.ini.dumps(node.metadata.get('mariadb/conf')), + 'content_type': 'text', }, } diff --git a/libs/ini.py b/libs/ini.py index f042569..f530144 100644 --- a/libs/ini.py +++ b/libs/ini.py @@ -14,7 +14,7 @@ class CaseSensitiveConfigParser(ConfigParser): return value def parse(text): - config = CaseSensitiveConfigParser() + config = CaseSensitiveConfigParser(allow_no_value=True) config.read_string(text) return { @@ -24,8 +24,7 @@ def parse(text): def dumps(dict): sorted_dict = json.loads(json.dumps(dict, sort_keys=True, cls=MetadataJSONEncoder)) - - parser = CaseSensitiveConfigParser() + parser = CaseSensitiveConfigParser(allow_no_value=True) parser.read_dict(sorted_dict) writable = Writable() -- 2.39.5 From 23947bd96769498c86725eb5c3c69af34d811409 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Fri, 27 Jun 2025 23:05:21 +0200 Subject: [PATCH 03/15] mariadb fixed --- bundles/mariadb/items.py | 2 -- bundles/mariadb/metadata.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/bundles/mariadb/items.py b/bundles/mariadb/items.py index 7ff4e83..6758c48 100644 --- a/bundles/mariadb/items.py +++ b/bundles/mariadb/items.py @@ -10,8 +10,6 @@ directories = { 'group': 'mysql', 'needs': [ 'zfs_dataset:tank/mariadb', - ], - 'needs': [ 'pkg_apt:mariadb-server', 'pkg_apt:mariadb-client', ], diff --git a/bundles/mariadb/metadata.py b/bundles/mariadb/metadata.py index 8b65e31..ee406cb 100644 --- a/bundles/mariadb/metadata.py +++ b/bundles/mariadb/metadata.py @@ -3,12 +3,12 @@ defaults = { 'packages': { 'mariadb-server': { 'needs': { - #'zfs_dataset:tank/mariadb', + 'zfs_dataset:tank/mariadb', }, }, 'mariadb-client': { 'needs': { - #'zfs_dataset:tank/mariadb', + 'zfs_dataset:tank/mariadb', }, }, }, -- 2.39.5 From df38fdb99ebd55d227c497bed354b297bd22ac11 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Fri, 27 Jun 2025 23:05:26 +0200 Subject: [PATCH 04/15] new router --- nodes/home.router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodes/home.router.py b/nodes/home.router.py index b4ec4fc..12169fd 100644 --- a/nodes/home.router.py +++ b/nodes/home.router.py @@ -15,7 +15,7 @@ 'id': '1d6a43e5-858c-42f9-9c40-ab63d61c787c', 'network': { 'external': { - 'interface': 'enx00e04c220682', + 'interface': 'enp2s0', 'ipv4': '10.0.99.126/24', 'gateway4': '10.0.99.1', 'vlans': {'iot', 'internet', 'guest', 'rolf', 'internal', 'proxmox'}, -- 2.39.5 From 800bd90778fc6b60c81c56b6043c8d13820ea9c1 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sat, 28 Jun 2025 21:56:11 +0200 Subject: [PATCH 05/15] remove apcupsd --- nodes/home.server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nodes/home.server.py b/nodes/home.server.py index ecc3253..4b30c61 100644 --- a/nodes/home.server.py +++ b/nodes/home.server.py @@ -12,7 +12,6 @@ 'build-server', ], 'bundles': [ - 'apcupsd', 'build-agent', 'crystal', 'gitea', -- 2.39.5 From 8d941ebef4cf9c55ae287387b555d7e334406f57 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sat, 28 Jun 2025 21:57:30 +0200 Subject: [PATCH 06/15] open fw for iperf --- groups/os/debian.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/groups/os/debian.py b/groups/os/debian.py index 69bb0d2..270fa60 100644 --- a/groups/os/debian.py +++ b/groups/os/debian.py @@ -47,6 +47,14 @@ 'mtr-tiny': {}, }, }, + # iperf3 + 'nftables': { + 'input': { + 'tcp dport 5201 accept', + 'udp dport 5201 accept', + }, + }, + }, 'os': 'debian', 'pip_command': 'pip3', -- 2.39.5 From 6e5ce8581bf2cf2409a6cda7e8273ee4195c6972 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Thu, 5 Jun 2025 18:13:54 +0200 Subject: [PATCH 07/15] wip --- bundles/systemd-networkd/items.py | 3 --- nodes/home.server.py | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/bundles/systemd-networkd/items.py b/bundles/systemd-networkd/items.py index af033ba..8d6e77a 100644 --- a/bundles/systemd-networkd/items.py +++ b/bundles/systemd-networkd/items.py @@ -1,9 +1,6 @@ assert node.has_bundle('systemd') files = { - '/etc/network/interfaces': { - 'delete': True, - }, '/etc/resolv.conf': { 'content_type': 'mako', }, diff --git a/nodes/home.server.py b/nodes/home.server.py index 4b30c61..cc6f05a 100644 --- a/nodes/home.server.py +++ b/nodes/home.server.py @@ -33,6 +33,7 @@ 'raspberrymatic-cert', 'wol-waker', 'zfs', + 'proxmox-ve', ], 'metadata': { 'id': 'af96709e-b13f-4965-a588-ef2cd476437a', -- 2.39.5 From 33d6888af4f9e10d5d0c814af60e027fff604b0a Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sun, 22 Jun 2025 14:00:59 +0200 Subject: [PATCH 08/15] mailman wip --- bundles/mailman/files/postfix.cf | 56 ++++++++++++++++++++++++++ bundles/mailman/items.py | 34 ++++++++++++++++ bundles/mailman/metadata.py | 29 ++++++++++++++ data/mailman/vhost.conf | 31 +++++++++++++++ nodes/htz.mails.py | 5 +++ nodes/mseibert.mailman.py | 67 ++++++++++++++++++++++++++++++++ 6 files changed, 222 insertions(+) create mode 100644 bundles/mailman/files/postfix.cf create mode 100644 bundles/mailman/items.py create mode 100644 bundles/mailman/metadata.py create mode 100644 data/mailman/vhost.conf create mode 100644 nodes/mseibert.mailman.py diff --git a/bundles/mailman/files/postfix.cf b/bundles/mailman/files/postfix.cf new file mode 100644 index 0000000..4060ed2 --- /dev/null +++ b/bundles/mailman/files/postfix.cf @@ -0,0 +1,56 @@ +# See /usr/share/postfix/main.cf.dist for a commented, more complete version + + +# Debian specific: Specifying a file name will cause the first +# line of that file to be used as the name. The Debian default +# is /etc/mailname. +#myorigin = /etc/mailname + +smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU) +biff = no + +# appending .domain is the MUA's job. +append_dot_mydomain = no + +# Uncomment the next line to generate "delayed mail" warnings +#delay_warning_time = 4h + +readme_directory = no + +# See http://www.postfix.org/COMPATIBILITY_README.html -- default to 3.6 on +# fresh installs. +compatibility_level = 3.6 + + + +# TLS parameters +smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem +smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key +smtpd_tls_security_level=may + +smtp_tls_CApath=/etc/ssl/certs +smtp_tls_security_level=may +smtp_tls_session_cache_database = <%text>btree:${data_directory}/smtp_scache + + +smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination +myhostname = ${hostname} +alias_maps = hash:/etc/aliases +alias_database = hash:/etc/aliases +mydestination = $myhostname, localhost, localhost.localdomain, ${hostname} +relayhost = +mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 +mailbox_size_limit = 0 +recipient_delimiter = + +inet_interfaces = all +inet_protocols = all + +unknown_local_recipient_reject_code = 550 +owner_request_special = no + +transport_maps = + hash:/opt/mailman/mm/var/data/postfix_lmtp +local_recipient_maps = + hash:/opt/mailman/mm/var/data/postfix_lmtp +relay_domains = + hash:/opt/mailman/mm/var/data/postfix_domains diff --git a/bundles/mailman/items.py b/bundles/mailman/items.py new file mode 100644 index 0000000..deae9a6 --- /dev/null +++ b/bundles/mailman/items.py @@ -0,0 +1,34 @@ +files = { + '/etc/postfix/main.cf': { + 'source': 'postfix.cf', + 'content_type': 'mako', + 'mode': '0644', + 'context': { + 'hostname': node.metadata.get('mailman/hostname'), + }, + 'needs': { + 'pkg_apt:postfix', + }, + 'triggers': { + 'svc_systemd:postfix.service:restart', + }, + } +} + +svc_systemd = { + 'postfix.service': { + 'needs': { + 'pkg_apt:postfix', + }, + }, + 'mailman3.service': { + 'needs': { + 'pkg_apt:mailman3-full', + }, + }, + 'mailman3-web.service': { + 'needs': { + 'pkg_apt:mailman3-full', + }, + }, +} diff --git a/bundles/mailman/metadata.py b/bundles/mailman/metadata.py new file mode 100644 index 0000000..cf223c4 --- /dev/null +++ b/bundles/mailman/metadata.py @@ -0,0 +1,29 @@ +defaults = { + 'apt': { + 'packages': { + 'mailman3-full': {}, + 'postfix': {}, + 'apache2': { + 'installed': False, + 'needs': { + 'pkg_apt:mailman3-full', + }, + }, + }, + }, +} + + +@metadata_reactor.provides( + 'nginx/vhosts', +) +def nginx(metadata): + return { + 'nginx': { + 'vhosts': { + metadata.get('mailman/hostname'): { + 'content': 'mailman/vhost.conf', + }, + }, + }, + } diff --git a/data/mailman/vhost.conf b/data/mailman/vhost.conf new file mode 100644 index 0000000..6fc4af4 --- /dev/null +++ b/data/mailman/vhost.conf @@ -0,0 +1,31 @@ +upstream mailman3 { + server unix:/run/mailman3-web/uwsgi.sock fail_timeout=0; +} + +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name ${server_name}; + + ssl_certificate /var/lib/dehydrated/certs/${server_name}/fullchain.pem; + ssl_certificate_key /var/lib/dehydrated/certs/${server_name}/privkey.pem; + + server_tokens off; + + location / { + uwsgi_pass mailman3; + include /etc/nginx/params/uwsgi; + } + + location /mailman3/static { + alias /var/lib/mailman3/web/static; + } + + location /mailman3/static/favicon.ico { + alias /var/lib/mailman3/web/static/postorius/img/favicon.ico; + } + +# return 301 https://$server_name$request_uri; + access_log /var/log/nginx/mailman3/access.log combined; + error_log /var/log/nginx/mailman3/error.log; +} \ No newline at end of file diff --git a/nodes/htz.mails.py b/nodes/htz.mails.py index c033591..239c565 100644 --- a/nodes/htz.mails.py +++ b/nodes/htz.mails.py @@ -200,6 +200,11 @@ '10.0.228.0/24', ], }, + 'mseibert.mailman': { + 'allowed_ips': [ + '10.0.229.0/24', + ], + }, }, 'clients': { 'macbook': { diff --git a/nodes/mseibert.mailman.py b/nodes/mseibert.mailman.py new file mode 100644 index 0000000..ef9cd1b --- /dev/null +++ b/nodes/mseibert.mailman.py @@ -0,0 +1,67 @@ +# https://teamvault.apps.seibert-media.net/secrets/mkqMRv/ +# https://console.hetzner.cloud/projects/889138/servers/46578341 + +# mailman.ckn.li + +{ + 'hostname': '91.99.123.176', + 'groups': [ + #'backup', + 'debian-12', + #'monitored', + 'webserver', + ], + 'bundles': [ + 'mailman', + #'wireguard', + #'zfs', + ], + 'metadata': { + 'id': '854cb39f-d964-4cc7-9051-ba6574708820', + 'network': { + 'internal': { + 'interface': 'ens10', + 'ipv4': '10.0.229.2/24', + }, + 'external': { + 'interface': 'eth0', + 'ipv4': '91.99.123.176/32', + 'gateway4': '172.31.1.1', + 'ipv6': '2a01:4f8:c013:2030::2/64', + 'gateway6': 'fe80::1', + }, + }, + 'mailman': { + 'hostname': 'mailman.ckn.li', + }, + 'overwrite_nameservers': [ + '8.8.8.8', + ], + 'vm': { + 'cores': 2, + 'ram': 4096, + }, + 'wireguard': { + 'my_ip': '172.30.0.240/32', + 's2s': { + 'htz.mails': { + 'allowed_ips': [ + '10.0.0.0/24', + '10.0.2.0/24', + '10.0.9.0/24', + '10.0.10.0/24', + ], + }, + }, + }, + 'zfs': { + 'pools': { + 'tank': { + 'devices': [ + '/var/lib/zfs_file', + ], + }, + }, + }, + }, +} -- 2.39.5 From 76cee836b964d7694711d6839efd5b7039c91b62 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sun, 22 Jun 2025 14:38:50 +0200 Subject: [PATCH 09/15] wip --- bundles/mailman/items.py | 15 +++++++++++++++ bundles/mailman/metadata.py | 7 +++++++ groups/applications/dnsserver.py | 4 ++-- nodes/home.router.py | 2 ++ nodes/mseibert.mailman.py | 4 ++-- 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/bundles/mailman/items.py b/bundles/mailman/items.py index deae9a6..c539b9e 100644 --- a/bundles/mailman/items.py +++ b/bundles/mailman/items.py @@ -1,3 +1,18 @@ +directories = { + '/var/lib/mailman3': { + 'owner': 'list', + 'group': 'list', + 'needs': { + 'zfs_dataset:tank/mailman', + 'pkg_apt:mailman3-full', + }, + 'needed_by': { + 'svc_systemd:mailman3.service', + 'svc_systemd:mailman3-web.service', + }, + }, +} + files = { '/etc/postfix/main.cf': { 'source': 'postfix.cf', diff --git a/bundles/mailman/metadata.py b/bundles/mailman/metadata.py index cf223c4..1b2c57b 100644 --- a/bundles/mailman/metadata.py +++ b/bundles/mailman/metadata.py @@ -11,6 +11,13 @@ defaults = { }, }, }, + 'zfs': { + 'datasets': { + 'tank/mailman': { + 'mountpoint': '/var/lib/mailman3', + }, + }, + }, } diff --git a/groups/applications/dnsserver.py b/groups/applications/dnsserver.py index 68c889c..3e4edf8 100644 --- a/groups/applications/dnsserver.py +++ b/groups/applications/dnsserver.py @@ -1,5 +1,5 @@ { - 'bundles': [ + 'bundles': { 'bind', - ], + }, } diff --git a/nodes/home.router.py b/nodes/home.router.py index 12169fd..bb9a888 100644 --- a/nodes/home.router.py +++ b/nodes/home.router.py @@ -83,6 +83,8 @@ '10.0.10.0/24', #'192.168.179.0/24', # while raspi at home '10.0.227.0/24', # mseibert.freescout + '10.0.228.0/24', # mseibert.yourls + '10.0.229.0/24', # mseibert.mailsman ], }, }, diff --git a/nodes/mseibert.mailman.py b/nodes/mseibert.mailman.py index ef9cd1b..6ea4d44 100644 --- a/nodes/mseibert.mailman.py +++ b/nodes/mseibert.mailman.py @@ -13,8 +13,8 @@ ], 'bundles': [ 'mailman', - #'wireguard', - #'zfs', + 'wireguard', + 'zfs', ], 'metadata': { 'id': '854cb39f-d964-4cc7-9051-ba6574708820', -- 2.39.5 From 29ac3d3dd7ab72f9f1df7881871f387db9501291 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sun, 22 Jun 2025 16:43:44 +0200 Subject: [PATCH 10/15] wip --- bundles/mailman/files/mailman-hyperkitty.cfg | 22 ++ bundles/mailman/files/mailman-web.py | 188 +++++++++++++ bundles/mailman/files/mailman.cfg | 271 +++++++++++++++++++ bundles/mailman/files/uwsgi.ini | 50 ++++ bundles/mailman/items.py | 57 +++- bundles/mailman/metadata.py | 82 +++++- bundles/postgresql/files/pg_hba.conf | 22 ++ bundles/postgresql/items.py | 14 + nodes/mseibert.mailman.py | 2 + 9 files changed, 706 insertions(+), 2 deletions(-) create mode 100644 bundles/mailman/files/mailman-hyperkitty.cfg create mode 100644 bundles/mailman/files/mailman-web.py create mode 100644 bundles/mailman/files/mailman.cfg create mode 100644 bundles/mailman/files/uwsgi.ini create mode 100644 bundles/postgresql/files/pg_hba.conf diff --git a/bundles/mailman/files/mailman-hyperkitty.cfg b/bundles/mailman/files/mailman-hyperkitty.cfg new file mode 100644 index 0000000..dee9cbb --- /dev/null +++ b/bundles/mailman/files/mailman-hyperkitty.cfg @@ -0,0 +1,22 @@ +# This is the mailman extension configuration file to enable HyperKitty as an +# archiver. Remember to add the following lines in the mailman.cfg file: +# +# [archiver.hyperkitty] +# class: mailman_hyperkitty.Archiver +# enable: yes +# configuration: /etc/mailman3/mailman-hyperkitty.cfg +# + +[general] + +# This is your HyperKitty installation, preferably on the localhost. This +# address will be used by Mailman to forward incoming emails to HyperKitty +# for archiving. It does not need to be publicly available, in fact it's +# better if it is not. +# However, if your Mailman installation is accessed via HTTPS, the URL needs +# to match your SSL certificate (e.g. https://lists.example.com/hyperkitty). +base_url: http://${hostname}/mailman3/hyperkitty/ + +# The shared api_key, must be identical except for quoting to the value of +# MAILMAN_ARCHIVER_KEY in HyperKitty's settings. +api_key: ${archiver_key} diff --git a/bundles/mailman/files/mailman-web.py b/bundles/mailman/files/mailman-web.py new file mode 100644 index 0000000..cae7697 --- /dev/null +++ b/bundles/mailman/files/mailman-web.py @@ -0,0 +1,188 @@ +# This file is imported by the Mailman Suite. It is used to override +# the default settings from /usr/share/mailman3-web/settings.py. + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = '${secret_key}' + +ADMINS = ( + ('Mailman Suite Admin', 'root@localhost'), +) + +# Hosts/domain names that are valid for this site; required if DEBUG is False +# See https://docs.djangoproject.com/en/1.8/ref/settings/#allowed-hosts +# Set to '*' per default in the Deian package to allow all hostnames. Mailman3 +# is meant to run behind a webserver reverse proxy anyway. +ALLOWED_HOSTS = [ + '${hostname}', +] + +# Mailman API credentials +MAILMAN_REST_API_URL = 'http://localhost:8001' +MAILMAN_REST_API_USER = 'restadmin' +MAILMAN_REST_API_PASS = '${api_password}' +MAILMAN_ARCHIVER_KEY = '${archiver_key}' +MAILMAN_ARCHIVER_FROM = ('127.0.0.1', '::1') + +# Application definition + +INSTALLED_APPS = ( + 'hyperkitty', + 'postorius', + 'django_mailman3', + # Uncomment the next line to enable the admin: + 'django.contrib.admin', + # Uncomment the next line to enable admin documentation: + # 'django.contrib.admindocs', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.sites', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'rest_framework', + 'django_gravatar', + 'compressor', + 'haystack', + 'django_extensions', + 'django_q', + 'allauth', + 'allauth.account', + 'allauth.socialaccount', + 'django_mailman3.lib.auth.fedora', + #'allauth.socialaccount.providers.openid', + #'allauth.socialaccount.providers.github', + #'allauth.socialaccount.providers.gitlab', + #'allauth.socialaccount.providers.google', + #'allauth.socialaccount.providers.facebook', + #'allauth.socialaccount.providers.twitter', + #'allauth.socialaccount.providers.stackexchange', +) + + +# Database +# https://docs.djangoproject.com/en/1.8/ref/settings/#databases + +DATABASES = { + 'default': { + # Use 'sqlite3', 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. + #'ENGINE': 'django.db.backends.sqlite3', + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + #'ENGINE': 'django.db.backends.mysql', + # DB name or path to database file if using sqlite3. + #'NAME': '/var/lib/mailman3/web/mailman3web.db', + 'NAME': 'mailman', + # The following settings are not used with sqlite3: + 'USER': 'mailman', + 'PASSWORD': '${db_password}', + # HOST: empty for localhost through domain sockets or '127.0.0.1' for + # localhost through TCP. + 'HOST': '127.0.0.1', + # PORT: set to empty string for default. + 'PORT': '5432', + # OPTIONS: Extra parameters to use when connecting to the database. + 'OPTIONS': { + # Set sql_mode to 'STRICT_TRANS_TABLES' for MySQL. See + # https://docs.djangoproject.com/en/1.11/ref/ + # databases/#setting-sql-mode + #'init_command': "SET sql_mode='STRICT_TRANS_TABLES'", + }, + } +} + + +# If you're behind a proxy, use the X-Forwarded-Host header +# See https://docs.djangoproject.com/en/1.8/ref/settings/#use-x-forwarded-host +USE_X_FORWARDED_HOST = True + +# And if your proxy does your SSL encoding for you, set SECURE_PROXY_SSL_HEADER +# https://docs.djangoproject.com/en/1.8/ref/settings/#secure-proxy-ssl-header +# SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') +# SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_SCHEME', 'https') + +# Other security settings +# SECURE_SSL_REDIRECT = True +# If you set SECURE_SSL_REDIRECT to True, make sure the SECURE_REDIRECT_EXEMPT +# contains at least this line: +# SECURE_REDIRECT_EXEMPT = [ +# "archives/api/mailman/.*", # Request from Mailman. +# ] +# SESSION_COOKIE_SECURE = True +# SECURE_CONTENT_TYPE_NOSNIFF = True +# SECURE_BROWSER_XSS_FILTER = True +# CSRF_COOKIE_SECURE = True +# CSRF_COOKIE_HTTPONLY = True +# X_FRAME_OPTIONS = 'DENY' + + +# Internationalization +# https://docs.djangoproject.com/en/1.8/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True +USE_L10N = True +USE_TZ = True + + +# Set default domain for email addresses. +EMAILNAME = 'localhost.local' + +# If you enable internal authentication, this is the address that the emails +# will appear to be coming from. Make sure you set a valid domain name, +# otherwise the emails may get rejected. +# https://docs.djangoproject.com/en/1.8/ref/settings/#default-from-email +# DEFAULT_FROM_EMAIL = "mailing-lists@you-domain.org" +DEFAULT_FROM_EMAIL = 'postorius@{}'.format(EMAILNAME) + +# If you enable email reporting for error messages, this is where those emails +# will appear to be coming from. Make sure you set a valid domain name, +# otherwise the emails may get rejected. +# https://docs.djangoproject.com/en/1.8/ref/settings/#std:setting-SERVER_EMAIL +# SERVER_EMAIL = 'root@your-domain.org' +SERVER_EMAIL = 'root@{}'.format(EMAILNAME) + + +# Django Allauth +ACCOUNT_DEFAULT_HTTP_PROTOCOL = "https" + + +# +# Social auth +# +SOCIALACCOUNT_PROVIDERS = { + #'openid': { + # 'SERVERS': [ + # dict(id='yahoo', + # name='Yahoo', + # openid_url='http://me.yahoo.com'), + # ], + #}, + #'google': { + # 'SCOPE': ['profile', 'email'], + # 'AUTH_PARAMS': {'access_type': 'online'}, + #}, + #'facebook': { + # 'METHOD': 'oauth2', + # 'SCOPE': ['email'], + # 'FIELDS': [ + # 'email', + # 'name', + # 'first_name', + # 'last_name', + # 'locale', + # 'timezone', + # ], + # 'VERSION': 'v2.4', + #}, +} + +# On a production setup, setting COMPRESS_OFFLINE to True will bring a +# significant performance improvement, as CSS files will not need to be +# recompiled on each requests. It means running an additional "compress" +# management command after each code upgrade. +# http://django-compressor.readthedocs.io/en/latest/usage/#offline-compression +COMPRESS_OFFLINE = True + +POSTORIUS_TEMPLATE_BASE_URL = 'http://${hostname}/mailman3/' diff --git a/bundles/mailman/files/mailman.cfg b/bundles/mailman/files/mailman.cfg new file mode 100644 index 0000000..f1266b7 --- /dev/null +++ b/bundles/mailman/files/mailman.cfg @@ -0,0 +1,271 @@ +# Copyright (C) 2008-2017 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +# This file contains the Debian configuration for mailman. It uses ini-style +# formats under the lazr.config regime to define all system configuration +# options. See for details. + + +[mailman] +# This address is the "site owner" address. Certain messages which must be +# delivered to a human, but which can't be delivered to a list owner (e.g. a +# bounce from a list owner), will be sent to this address. It should point to +# a human. +site_owner: ${site_owner_email} + +# This is the local-part of an email address used in the From field whenever a +# message comes from some entity to which there is no natural reply recipient. +# Mailman will append '@' and the host name of the list involved. This +# address must not bounce and it must not point to a Mailman process. +noreply_address: noreply + +# The default language for this server. +default_language: de + +# Membership tests for posting purposes are usually performed by looking at a +# set of headers, passing the test if any of their values match a member of +# the list. Headers are checked in the order given in this variable. The +# value From_ means to use the envelope sender. Field names are case +# insensitive. This is a space separate list of headers. +sender_headers: from from_ reply-to sender + +# Mail command processor will ignore mail command lines after designated max. +email_commands_max_lines: 10 + +# Default length of time a pending request is live before it is evicted from +# the pending database. +pending_request_life: 3d + +# How long should files be saved before they are evicted from the cache? +cache_life: 7d + +# A callable to run with no arguments early in the initialization process. +# This runs before database initialization. +pre_hook: + +# A callable to run with no arguments late in the initialization process. +# This runs after adapters are initialized. +post_hook: + +# Which paths.* file system layout to use. +# You should not change this variable. +layout: debian + +# Can MIME filtered messages be preserved by list owners? +filtered_messages_are_preservable: no + +# How should text/html parts be converted to text/plain when the mailing list +# is set to convert HTML to plaintext? This names a command to be called, +# where the substitution variable $filename is filled in by Mailman, and +# contains the path to the temporary file that the command should read from. +# The command should print the converted text to stdout. +html_to_plain_text_command: /usr/bin/lynx -dump $filename + +# Specify what characters are allowed in list names. Characters outside of +# the class [-_.+=!$*{}~0-9a-z] matched case insensitively are never allowed, +# but this specifies a subset as the only allowable characters. This must be +# a valid character class regexp or the effect on list creation is +# unpredictable. +listname_chars: [-_.0-9a-z] + + +[shell] +# `mailman shell` (also `withlist`) gives you an interactive prompt that you +# can use to interact with an initialized and configured Mailman system. Use +# --help for more information. This section allows you to configure certain +# aspects of this interactive shell. + +# Customize the interpreter prompt. +prompt: >>> + +# Banner to show on startup. +banner: Welcome to the GNU Mailman shell + +# Use IPython as the shell, which must be found on the system. Valid values +# are `no`, `yes`, and `debug` where the latter is equivalent to `yes` except +# that any import errors will be displayed to stderr. +use_ipython: no + +# Set this to allow for command line history if readline is available. This +# can be as simple as $var_dir/history.py to put the file in the var directory. +history_file: + + +[paths.debian] +# Important directories for Mailman operation. These are defined here so that +# different layouts can be supported. For example, a developer layout would +# be different from a FHS layout. Most paths are based off the var_dir, and +# often just setting that will do the right thing for all the other paths. +# You might also have to set spool_dir though. +# +# Substitutions are allowed, but must be of the form $var where 'var' names a +# configuration variable in the paths.* section. Substitutions are expanded +# recursively until no more $-variables are present. Beware of infinite +# expansion loops! +# +# This is the root of the directory structure that Mailman will use to store +# its run-time data. +var_dir: /var/lib/mailman3 +# This is where the Mailman queue files directories will be created. +queue_dir: $var_dir/queue +# This is the directory containing the Mailman 'runner' and 'master' commands +# if set to the string '$argv', it will be taken as the directory containing +# the 'mailman' command. +bin_dir: /usr/lib/mailman3/bin +# All list-specific data. +list_data_dir: $var_dir/lists +# Directory where log files go. +log_dir: /var/log/mailman3 +# Directory for system-wide locks. +lock_dir: $var_dir/locks +# Directory for system-wide data. +data_dir: $var_dir/data +# Cache files. +cache_dir: $var_dir/cache +# Directory for configuration files and such. +etc_dir: /etc/mailman3 +# Directory containing Mailman plugins. +ext_dir: $var_dir/ext +# Directory where the default IMessageStore puts its messages. +messages_dir: $var_dir/messages +# Directory for archive backends to store their messages in. Archivers should +# create a subdirectory in here to store their files. +archive_dir: $var_dir/archives +# Root directory for site-specific template override files. +template_dir: $var_dir/templates +# There are also a number of paths to specific file locations that can be +# defined. For these, the directory containing the file must already exist, +# or be one of the directories created by Mailman as per above. +# +# This is where PID file for the master runner is stored. +pid_file: /run/mailman3/master.pid +# Lock file. +lock_file: $lock_dir/master.lck + + +[database] +# The class implementing the IDatabase. +class: mailman.database.sqlite.SQLiteDatabase +#class: mailman.database.mysql.MySQLDatabase +#class: mailman.database.postgresql.PostgreSQLDatabase + +# Use this to set the Storm database engine URL. You generally have one +# primary database connection for all of Mailman. List data and most rosters +# will store their data in this database, although external rosters may access +# other databases in their own way. This string supports standard +# 'configuration' substitutions. +url: sqlite:///$DATA_DIR/mailman.db +#url: mysql+pymysql://mailman3:mmpass@localhost/mailman3?charset=utf8&use_unicode=1 +#url: postgresql://mailman3:mmpass@localhost/mailman3 + +debug: no + + +[logging.debian] +# This defines various log settings. The options available are: +# +# - level -- Overrides the default level; this may be any of the +# standard Python logging levels, case insensitive. +# - format -- Overrides the default format string +# - datefmt -- Overrides the default date format string +# - path -- Overrides the default logger path. This may be a relative +# path name, in which case it is relative to Mailman's LOG_DIR, +# or it may be an absolute path name. You cannot change the +# handler class that will be used. +# - propagate -- Boolean specifying whether to propagate log message from this +# logger to the root "mailman" logger. You cannot override +# settings for the root logger. +# +# In this section, you can define defaults for all loggers, which will be +# prefixed by 'mailman.'. Use subsections to override settings for specific +# loggers. The names of the available loggers are: +# +# - archiver -- All archiver output +# - bounce -- All bounce processing logs go here +# - config -- Configuration issues +# - database -- Database logging (SQLAlchemy and Alembic) +# - debug -- Only used for development +# - error -- All exceptions go to this log +# - fromusenet -- Information related to the Usenet to Mailman gateway +# - http -- Internal wsgi-based web interface +# - locks -- Lock state changes +# - mischief -- Various types of hostile activity +# - runner -- Runner process start/stops +# - smtp -- Successful SMTP activity +# - smtp-failure -- Unsuccessful SMTP activity +# - subscribe -- Information about leaves/joins +# - vette -- Message vetting information +format: %(asctime)s (%(process)d) %(message)s +datefmt: %b %d %H:%M:%S %Y +propagate: no +level: info +path: mailman.log + +[webservice] +# The hostname at which admin web service resources are exposed. +hostname: ${hostname} + +# The port at which the admin web service resources are exposed. +port: 8001 + +# Whether or not requests to the web service are secured through SSL. +use_https: no + +# Whether or not to show tracebacks in an HTTP response for a request that +# raised an exception. +show_tracebacks: yes + +# The API version number for the current (highest) API. +api_version: 3.1 + +# The administrative username. +admin_user: restadmin + +# The administrative password. +admin_pass: ${api_password} + +[mta] +# The class defining the interface to the incoming mail transport agent. +#incoming: mailman.mta.exim4.LMTP +incoming: mailman.mta.postfix.LMTP + +# The callable implementing delivery to the outgoing mail transport agent. +# This must accept three arguments, the mailing list, the message, and the +# message metadata dictionary. +outgoing: mailman.mta.deliver.deliver + +# How to connect to the outgoing MTA. If smtp_user and smtp_pass is given, +# then Mailman will attempt to log into the MTA when making a new connection. +smtp_host: localhost +smtp_port: 25 +smtp_user: +smtp_pass: + +# Where the LMTP server listens for connections. Use 127.0.0.1 instead of +# localhost for Postfix integration, because Postfix only consults DNS +# (e.g. not /etc/hosts). +lmtp_host: 127.0.0.1 +lmtp_port: 8024 + +# Where can we find the mail server specific configuration file? The path can +# be either a file system path or a Python import path. If the value starts +# with python: then it is a Python import path, otherwise it is a file system +# path. File system paths must be absolute since no guarantees are made about +# the current working directory. Python paths should not include the trailing +# .cfg, which the file must end with. +#configuration: python:mailman.config.exim4 +configuration: python:mailman.config.postfix diff --git a/bundles/mailman/files/uwsgi.ini b/bundles/mailman/files/uwsgi.ini new file mode 100644 index 0000000..57e10ac --- /dev/null +++ b/bundles/mailman/files/uwsgi.ini @@ -0,0 +1,50 @@ +[uwsgi] +# Port on which uwsgi will be listening. +uwsgi-socket = /run/mailman3-web/uwsgi.sock + +#Enable threading for python +enable-threads = true + +# Move to the directory wher the django files are. +chdir = /usr/share/mailman3-web + +# Use the wsgi file provided with the django project. +wsgi-file = wsgi.py + +# Setup default number of processes and threads per process. +master = true +process = 2 +threads = 2 + +# Drop privielges and don't run as root. +uid = www-data +gid = www-data + +plugins = python3 + +# Setup the django_q related worker processes. +attach-daemon = python3 manage.py qcluster + +# Setup hyperkitty's cron jobs. +#unique-cron = -1 -1 -1 -1 -1 ./manage.py runjobs minutely +#unique-cron = -15 -1 -1 -1 -1 ./manage.py runjobs quarter_hourly +#unique-cron = 0 -1 -1 -1 -1 ./manage.py runjobs hourly +#unique-cron = 0 0 -1 -1 -1 ./manage.py runjobs daily +#unique-cron = 0 0 1 -1 -1 ./manage.py runjobs monthly +#unique-cron = 0 0 -1 -1 0 ./manage.py runjobs weekly +#unique-cron = 0 0 1 1 -1 ./manage.py runjobs yearly + +# Setup the request log. +#req-logger = file:/var/log/mailman3/web/mailman-web.log + +# Log cron seperately. +#logger = cron file:/var/log/mailman3/web/mailman-web-cron.log +#log-route = cron uwsgi-cron + +# Log qcluster commands seperately. +#logger = qcluster file:/var/log/mailman3/web/mailman-web-qcluster.log +#log-route = qcluster uwsgi-daemons + +# Last log and it logs the rest of the stuff. +#logger = file:/var/log/mailman3/web/mailman-web-error.log +logto = /var/log/mailman3/web/mailman-web.log diff --git a/bundles/mailman/items.py b/bundles/mailman/items.py index c539b9e..296b1e7 100644 --- a/bundles/mailman/items.py +++ b/bundles/mailman/items.py @@ -27,7 +27,62 @@ files = { 'triggers': { 'svc_systemd:postfix.service:restart', }, - } + }, + '/etc/mailman3/mailman.cfg': { + 'content_type': 'mako', + 'owner': 'root', + 'group': 'list', + 'mode': '0640', + 'context': node.metadata.get('mailman'), + 'needs': { + 'pkg_apt:mailman3-full', + }, + 'triggers': { + 'svc_systemd:mailman3.service:restart', + 'svc_systemd:mailman3-web.service:restart', + }, + }, + '/etc/mailman3/mailman-web.py': { + 'content_type': 'mako', + 'owner': 'root', + 'group': 'www-data', + 'mode': '0640', + 'context': node.metadata.get('mailman'), + 'needs': { + 'pkg_apt:mailman3-full', + }, + 'triggers': { + 'svc_systemd:mailman3.service:restart', + 'svc_systemd:mailman3-web.service:restart', + }, + }, + '/etc/mailman3/mailman-hyperkitty.cfg': { + 'content_type': 'mako', + 'owner': 'root', + 'group': 'list', + 'mode': '0640', + 'context': node.metadata.get('mailman'), + 'needs': { + 'pkg_apt:mailman3-full', + }, + 'triggers': { + 'svc_systemd:mailman3.service:restart', + 'svc_systemd:mailman3-web.service:restart', + }, + }, + '/etc/mailman3/uwsgi.ini': { + 'content_type': 'text', + 'owner': 'root', + 'group': 'root', + 'mode': '0644', + 'needs': { + 'pkg_apt:mailman3-full', + }, + 'triggers': { + 'svc_systemd:mailman3.service:restart', + 'svc_systemd:mailman3-web.service:restart', + }, + }, } svc_systemd = { diff --git a/bundles/mailman/metadata.py b/bundles/mailman/metadata.py index 1b2c57b..dda398a 100644 --- a/bundles/mailman/metadata.py +++ b/bundles/mailman/metadata.py @@ -1,8 +1,29 @@ +import base64 + +def derive_mailadmin_secret(metadata, salt): + node_id = metadata.get('id') + raw = base64.b64decode( + repo.vault.random_bytes_as_base64_for(f'{node_id}_{salt}', length=32).value + ) + return base64.urlsafe_b64encode(raw).rstrip(b'=').decode('ascii') + + defaults = { 'apt': { 'packages': { - 'mailman3-full': {}, + 'mailman3-full': { + 'needs': { + 'postgres_db:mailman', + 'postgres_role:mailman', + 'zfs_dataset:tank/mailman', + } + }, 'postfix': {}, + 'python3-psycopg2': { + 'needed_by': { + 'pkg_apt:mailman3-full', + }, + }, 'apache2': { 'installed': False, 'needs': { @@ -21,6 +42,33 @@ defaults = { } +@metadata_reactor.provides( + 'postgresql', + 'mailman', +) +def postgresql(metadata): + node_id = metadata.get('id') + db_password = repo.vault.password_for(f'{node_id} database mailman') + + return { + 'postgresql': { + 'databases': { + 'mailman': { + 'owner': 'mailman', + }, + }, + 'roles': { + 'mailman': { + 'password': db_password, + }, + }, + }, + 'mailman': { + 'db_password': db_password, + }, + } + + @metadata_reactor.provides( 'nginx/vhosts', ) @@ -34,3 +82,35 @@ def nginx(metadata): }, }, } + + +@metadata_reactor.provides( + 'mailman/secret_key', +) +def secret_key(metadata): + import base64 + + node_id = metadata.get('id') + raw = base64.b64decode( + repo.vault.random_bytes_as_base64_for(f'{node_id}_mailman_secret_key', length=32).value + ) + secret_key = base64.urlsafe_b64encode(raw).rstrip(b'=').decode('ascii') + + return { + 'mailman': { + 'secret_key': secret_key, + }, + } + + +@metadata_reactor.provides( + 'mailman', +) +def secrets(metadata): + return { + 'mailman': { + 'web_secret': derive_mailadmin_secret(metadata, 'secret_key'), + 'api_password': derive_mailadmin_secret(metadata, 'api_password'), + 'archiver_key': derive_mailadmin_secret(metadata, 'archiver_key'), + }, + } diff --git a/bundles/postgresql/files/pg_hba.conf b/bundles/postgresql/files/pg_hba.conf new file mode 100644 index 0000000..801cef7 --- /dev/null +++ b/bundles/postgresql/files/pg_hba.conf @@ -0,0 +1,22 @@ +# DO NOT DISABLE! +# If you change this first entry you will need to make sure that the +# database superuser can access the database using some other method. +# Noninteractive access to all databases is required during automatic +# maintenance (custom daily cronjobs, replication, and similar tasks). +# +# Database administrative login by Unix domain socket +local all postgres peer + +# TYPE DATABASE USER ADDRESS METHOD + +# "local" is for Unix domain socket connections only +local all all peer +# IPv4 local connections: +host all all 127.0.0.1/32 md5 +# IPv6 local connections: +host all all ::1/128 md5 +# Allow replication connections from localhost, by a user with the +# replication privilege. +local replication all peer +host replication all 127.0.0.1/32 md5 +host replication all ::1/128 md5 diff --git a/bundles/postgresql/items.py b/bundles/postgresql/items.py index b7e7ba8..21ae0cc 100644 --- a/bundles/postgresql/items.py +++ b/bundles/postgresql/items.py @@ -18,6 +18,20 @@ directories = { } files = { + f"/etc/postgresql/{version}/main/pg_hba.conf": { + 'mode': '0640', + 'owner': 'postgres', + 'group': 'postgres', + 'needs': [ + 'pkg_apt:postgresql', + ], + 'needed_by': [ + 'svc_systemd:postgresql.service', + ], + 'triggers': [ + 'svc_systemd:postgresql.service:restart', + ], + }, f"/etc/postgresql/{version}/main/conf.d/managed.conf": { 'content': '\n'.join( f'{key} = {value}' diff --git a/nodes/mseibert.mailman.py b/nodes/mseibert.mailman.py index 6ea4d44..b2a9b7d 100644 --- a/nodes/mseibert.mailman.py +++ b/nodes/mseibert.mailman.py @@ -15,6 +15,7 @@ 'mailman', 'wireguard', 'zfs', + 'postgresql', ], 'metadata': { 'id': '854cb39f-d964-4cc7-9051-ba6574708820', @@ -33,6 +34,7 @@ }, 'mailman': { 'hostname': 'mailman.ckn.li', + 'site_owner_email': 'martin.seibert@seibert.group', }, 'overwrite_nameservers': [ '8.8.8.8', -- 2.39.5 From d59802ad9242d850682d5ea7a39e1e91b54690c6 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sun, 22 Jun 2025 16:48:22 +0200 Subject: [PATCH 11/15] wip --- bundles/mailman/files/mailman.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundles/mailman/files/mailman.cfg b/bundles/mailman/files/mailman.cfg index f1266b7..d294da4 100644 --- a/bundles/mailman/files/mailman.cfg +++ b/bundles/mailman/files/mailman.cfg @@ -217,7 +217,7 @@ path: mailman.log [webservice] # The hostname at which admin web service resources are exposed. -hostname: ${hostname} +hostname: localhost # The port at which the admin web service resources are exposed. port: 8001 -- 2.39.5 From 15562df71f7a60a729d5bbf680d0f1e50bd41707 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sun, 22 Jun 2025 17:22:36 +0200 Subject: [PATCH 12/15] wip --- bundles/postgresql/files/pg_hba.conf | 8 ++++---- bundles/postgresql/items.py | 1 + nodes/mseibert.freescout.py | 3 +++ nodes/mseibert.mailman.py | 4 ++-- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/bundles/postgresql/files/pg_hba.conf b/bundles/postgresql/files/pg_hba.conf index 801cef7..950dd28 100644 --- a/bundles/postgresql/files/pg_hba.conf +++ b/bundles/postgresql/files/pg_hba.conf @@ -12,11 +12,11 @@ local all postgres peer # "local" is for Unix domain socket connections only local all all peer # IPv4 local connections: -host all all 127.0.0.1/32 md5 +host all all 127.0.0.1/32 ${node.metadata.get('postgresql/password_algorithm', 'md5')} # IPv6 local connections: -host all all ::1/128 md5 +host all all ::1/128 ${node.metadata.get('postgresql/password_algorithm', 'md5')} # Allow replication connections from localhost, by a user with the # replication privilege. local replication all peer -host replication all 127.0.0.1/32 md5 -host replication all ::1/128 md5 +host replication all 127.0.0.1/32 ${node.metadata.get('postgresql/password_algorithm', 'md5')} +host replication all ::1/128 ${node.metadata.get('postgresql/password_algorithm', 'md5')} diff --git a/bundles/postgresql/items.py b/bundles/postgresql/items.py index 21ae0cc..db72238 100644 --- a/bundles/postgresql/items.py +++ b/bundles/postgresql/items.py @@ -19,6 +19,7 @@ directories = { files = { f"/etc/postgresql/{version}/main/pg_hba.conf": { + 'content_type': 'mako', 'mode': '0640', 'owner': 'postgres', 'group': 'postgres', diff --git a/nodes/mseibert.freescout.py b/nodes/mseibert.freescout.py index c186b08..57407fb 100644 --- a/nodes/mseibert.freescout.py +++ b/nodes/mseibert.freescout.py @@ -30,6 +30,9 @@ 'gateway6': 'fe80::1', }, }, + 'postgresql': { + 'password_algorithm': 'scram-sha-256', + }, 'freescout': { 'domain': 'foerderkreis.oranienschule-wiesbaden-wiki.de', }, diff --git a/nodes/mseibert.mailman.py b/nodes/mseibert.mailman.py index b2a9b7d..06c53ab 100644 --- a/nodes/mseibert.mailman.py +++ b/nodes/mseibert.mailman.py @@ -6,9 +6,9 @@ { 'hostname': '91.99.123.176', 'groups': [ - #'backup', + 'backup', 'debian-12', - #'monitored', + 'monitored', 'webserver', ], 'bundles': [ -- 2.39.5 From 9159cd1eec6f6b43ac6c0d03a1b9c73c4e92ec04 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sun, 22 Jun 2025 20:11:03 +0200 Subject: [PATCH 13/15] wip --- bundles/mailman/files/mailman-web.py | 2 ++ bundles/mailman/files/mailman.cfg | 6 ++++++ bundles/mailman/files/postfix.cf | 10 +++------- nodes/mseibert.mailman.py | 4 ++++ 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/bundles/mailman/files/mailman-web.py b/bundles/mailman/files/mailman-web.py index cae7697..baa8d73 100644 --- a/bundles/mailman/files/mailman-web.py +++ b/bundles/mailman/files/mailman-web.py @@ -1,3 +1,5 @@ +ACCOUNT_EMAIL_VERIFICATION='none' + # This file is imported by the Mailman Suite. It is used to override # the default settings from /usr/share/mailman3-web/settings.py. diff --git a/bundles/mailman/files/mailman.cfg b/bundles/mailman/files/mailman.cfg index d294da4..1b27fd2 100644 --- a/bundles/mailman/files/mailman.cfg +++ b/bundles/mailman/files/mailman.cfg @@ -250,6 +250,12 @@ outgoing: mailman.mta.deliver.deliver # How to connect to the outgoing MTA. If smtp_user and smtp_pass is given, # then Mailman will attempt to log into the MTA when making a new connection. +# smtp_host: smtp.ionos.de +# smtp_port: 587 +# smtp_user: ${smtp_user} +# smtp_pass: ${smtp_password} +# smtp_secure_mode: starttls + smtp_host: localhost smtp_port: 25 smtp_user: diff --git a/bundles/mailman/files/postfix.cf b/bundles/mailman/files/postfix.cf index 4060ed2..1521d9f 100644 --- a/bundles/mailman/files/postfix.cf +++ b/bundles/mailman/files/postfix.cf @@ -1,6 +1,5 @@ # See /usr/share/postfix/main.cf.dist for a commented, more complete version - # Debian specific: Specifying a file name will cause the first # line of that file to be used as the name. The Debian default # is /etc/mailname. @@ -21,8 +20,6 @@ readme_directory = no # fresh installs. compatibility_level = 3.6 - - # TLS parameters smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key @@ -32,7 +29,6 @@ smtp_tls_CApath=/etc/ssl/certs smtp_tls_security_level=may smtp_tls_session_cache_database = <%text>btree:${data_directory}/smtp_scache - smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination myhostname = ${hostname} alias_maps = hash:/etc/aliases @@ -49,8 +45,8 @@ unknown_local_recipient_reject_code = 550 owner_request_special = no transport_maps = - hash:/opt/mailman/mm/var/data/postfix_lmtp + hash:/var/lib/mailman3/data/postfix_lmtp local_recipient_maps = - hash:/opt/mailman/mm/var/data/postfix_lmtp + hash:/var/lib/mailman3/data/postfix_lmtp relay_domains = - hash:/opt/mailman/mm/var/data/postfix_domains + hash:/var/lib/mailman3/data/postfix_domains diff --git a/nodes/mseibert.mailman.py b/nodes/mseibert.mailman.py index 06c53ab..131ea23 100644 --- a/nodes/mseibert.mailman.py +++ b/nodes/mseibert.mailman.py @@ -35,6 +35,10 @@ 'mailman': { 'hostname': 'mailman.ckn.li', 'site_owner_email': 'martin.seibert@seibert.group', + 'smtp_host': 'smtp.ionos.de', + 'smtp_port': 465, + 'smtp_user': '!decrypt:encrypt$gAAAAABoWEcZlLxiTKluyg3gZ-un2fYkuviW9BD9tTW8mfKBL5d41Z1X7LtI5CDnhhLXTGFpPnY1thr17h22oW3Ybz_WPgvbJVepnVwmeQwvMpg2psATKAY=', + 'smtp_password': '!decrypt:encrypt$gAAAAABoWDusH3XY4ONh8MnmfBbyHW477ipjSycb3TiDGXxO5eujum80zXjNrOblswCGRTHsW9UasM_dXeeGBsa7KcK4s6AK_eynXCWeLCtXfrUSE_oEd7c=' }, 'overwrite_nameservers': [ '8.8.8.8', -- 2.39.5 From 1eafaf0d1c3f170da452e0aba2e61f51e0bb72fb Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sun, 22 Jun 2025 20:12:52 +0200 Subject: [PATCH 14/15] wip --- nodes/mseibert.mailman.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodes/mseibert.mailman.py b/nodes/mseibert.mailman.py index 131ea23..de15fcd 100644 --- a/nodes/mseibert.mailman.py +++ b/nodes/mseibert.mailman.py @@ -34,7 +34,7 @@ }, 'mailman': { 'hostname': 'mailman.ckn.li', - 'site_owner_email': 'martin.seibert@seibert.group', + 'site_owner_email': '!decrypt:encrypt$gAAAAABoWEeTyypfKw9l9jnNgF4GlS0-6O2NWCB0f3Fj1XnQ_HMjHXymAL8FWTyQjRmz3r8KnGJ-sogfnhW6lub_pnuk-wqB5Zuy9tgGsfi3RvkyNaOUeTE=', 'smtp_host': 'smtp.ionos.de', 'smtp_port': 465, 'smtp_user': '!decrypt:encrypt$gAAAAABoWEcZlLxiTKluyg3gZ-un2fYkuviW9BD9tTW8mfKBL5d41Z1X7LtI5CDnhhLXTGFpPnY1thr17h22oW3Ybz_WPgvbJVepnVwmeQwvMpg2psATKAY=', -- 2.39.5 From 6b9b74b1e189d7b544c5f440e5edf08e299a5fa6 Mon Sep 17 00:00:00 2001 From: CroneKorkN Date: Sun, 22 Jun 2025 21:04:43 +0200 Subject: [PATCH 15/15] wip --- bundles/mailman/files/mailman.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundles/mailman/files/mailman.cfg b/bundles/mailman/files/mailman.cfg index 1b27fd2..9141e5d 100644 --- a/bundles/mailman/files/mailman.cfg +++ b/bundles/mailman/files/mailman.cfg @@ -256,7 +256,7 @@ outgoing: mailman.mta.deliver.deliver # smtp_pass: ${smtp_password} # smtp_secure_mode: starttls -smtp_host: localhost +smtp_host: 127.0.0.1 smtp_port: 25 smtp_user: smtp_pass: -- 2.39.5