From c1e37b233c9723077434f3189bb482a6b4873a0f Mon Sep 17 00:00:00 2001
From: mwiegand <mwiegand@seibert-media.net>
Date: Sun, 13 Jun 2021 18:40:51 +0200
Subject: [PATCH] wip

---
 bundles/dovecot/files/dovecot-sql.conf |   5 +
 bundles/dovecot/files/dovecot.conf     | 139 +++++++++++++++++++++++++
 bundles/dovecot/items.py               |  56 ++++++++++
 bundles/dovecot/metadata.py            |  25 +++++
 bundles/mailserver/metadata.py         |  33 ++++++
 bundles/postfix/items.py               |  10 +-
 bundles/postfix/metadata.py            |  26 +----
 bundles/postgresql/items.py            |   5 -
 bundles/zfs/items.py                   |  41 ++++++++
 bundles/zfs/metadata.py                |  55 ++++++++++
 groups/applications/mailserver.py      |   3 +
 nodes/htz.mails.py                     |   7 ++
 12 files changed, 375 insertions(+), 30 deletions(-)
 create mode 100644 bundles/dovecot/files/dovecot-sql.conf
 create mode 100644 bundles/dovecot/files/dovecot.conf
 create mode 100644 bundles/dovecot/items.py
 create mode 100644 bundles/dovecot/metadata.py
 create mode 100644 bundles/mailserver/metadata.py
 create mode 100644 bundles/zfs/items.py
 create mode 100644 bundles/zfs/metadata.py

diff --git a/bundles/dovecot/files/dovecot-sql.conf b/bundles/dovecot/files/dovecot-sql.conf
new file mode 100644
index 0000000..bf7a023
--- /dev/null
+++ b/bundles/dovecot/files/dovecot-sql.conf
@@ -0,0 +1,5 @@
+connect = host=${host} dbname=${name} user=${user} password=${password}
+driver = pgsql
+default_pass_scheme = MD5-CRYPT
+password_query = SELECT username as user, password FROM mailbox WHERE username = '%u' AND active = true
+user_query = SELECT '/var/mail/vmail/' || maildir as home, 65534 as uid, 65534 as gid FROM mailbox WHERE username = '%u' AND active = true
diff --git a/bundles/dovecot/files/dovecot.conf b/bundles/dovecot/files/dovecot.conf
new file mode 100644
index 0000000..b59cfa0
--- /dev/null
+++ b/bundles/dovecot/files/dovecot.conf
@@ -0,0 +1,139 @@
+!include conf.d/*.conf
+
+namespace inbox {
+  separator = .
+  type = private
+  inbox = yes
+  location =
+  mailbox Drafts {
+    auto = subscribe
+    special_use = \Drafts
+  }
+  mailbox Junk {
+    auto = create
+    special_use = \Junk
+    autoexpunge = 30d
+  }
+  mailbox Sent {
+    auto = subscribe
+    special_use = \Sent
+  }
+  mailbox Trash {
+    auto = subscribe
+    special_use = \Trash
+    autoexpunge = 360d
+  }
+  prefix =
+}
+
+mail_location = maildir:/var/vmail/%u
+protocols = imap lmtp sieve
+
+ssl = yes
+ssl_cert = </var/lib/dehydrated/certs/${node.metadata.get('mailserver/hostname')}/fullchain.pem
+ssl_key = </var/lib/dehydrated/certs/${node.metadata.get('mailserver/hostname')}/privkey.pem
+ssl_dh = </etc/dovecot/ssl/dhparam.pem
+ssl_min_protocol = TLSv1.2
+ssl_cipher_list = EECDH+AESGCM:EDH+AESGCM
+ssl_prefer_server_ciphers = yes
+
+login_greeting = IMAPd ready
+auth_mechanisms = plain login
+first_valid_uid = 65534
+disable_plaintext_auth = yes
+mail_plugins = $mail_plugins zlib
+
+plugin {
+   zlib_save_level = 6
+   zlib_save = gz
+
+   sieve_plugins = sieve_imapsieve sieve_extprograms
+   sieve_dir = /var/vmail/sieve/%d/%n/
+   sieve = /var/vmail/sieve/%d/%n.sieve
+   sieve_pipe_bin_dir = /var/vmail/sieve/bin
+   sieve_extensions = +vnd.dovecot.pipe
+
+   old_stats_refresh = 30 secs
+   old_stats_track_cmds = yes
+
+% if node.has_bundle('rspamd'):
+   sieve_before = /var/vmail/sieve/global/spam-global.sieve
+
+   # From elsewhere to Spam folder
+   imapsieve_mailbox1_name = Junk
+   imapsieve_mailbox1_causes = COPY
+   imapsieve_mailbox1_before = file:/var/vmail/sieve/global/learn-spam.sieve
+
+   # From Spam folder to elsewhere
+   imapsieve_mailbox2_name = *
+   imapsieve_mailbox2_from = Junk
+   imapsieve_mailbox2_causes = COPY
+   imapsieve_mailbox2_before = file:/var/vmail/sieve/global/learn-ham.sieve
+% endif
+}
+
+service auth {
+   unix_listener /var/spool/postfix/private/auth {
+       mode = 0660
+       user = postfix
+       group = postfix
+   }
+
+   unix_listener auth-userdb {
+       mode = 0660
+       user = nobody
+       group = nogroup
+   }
+}
+
+service lmtp {
+   unix_listener /var/spool/postfix/private/dovecot-lmtp {
+       group = postfix
+       mode = 0600
+       user = postfix
+   }
+}
+
+service imap {
+   executable = imap
+}
+
+service imap-login {
+   service_count = 1
+   process_min_avail = 8
+   vsz_limit = 64M
+}
+
+service managesieve-login {
+   inet_listener sieve {
+       port = 4190
+   }
+}
+
+userdb {
+   driver = sql
+   args = /etc/dovecot/dovecot-sql.conf
+}
+
+passdb {
+   driver = sql
+   args = /etc/dovecot/dovecot-sql.conf
+}
+
+protocol lmtp {
+   mail_plugins = $mail_plugins sieve
+   postmaster_address = ${admin_email}
+}
+
+protocol imap {
+   mail_plugins = $mail_plugins imap_zlib imap_sieve imap_old_stats
+   mail_max_userip_connections = 50
+   imap_idle_notify_interval = 29 mins
+}
+
+protocol sieve {
+   plugin {
+       sieve = /var/vmail/sieve/%d/%n.sieve
+       sieve_storage = /var/vmail/sieve/%d/%n/
+   }
+}
diff --git a/bundles/dovecot/items.py b/bundles/dovecot/items.py
new file mode 100644
index 0000000..81caaf3
--- /dev/null
+++ b/bundles/dovecot/items.py
@@ -0,0 +1,56 @@
+assert node.has_bundle('postfix')
+assert node.has_bundle('postgresql')
+assert node.has_bundle('letsencrypt')
+
+directories = {
+    '/etc/dovecot/ssl': {},
+}
+
+files = {
+    '/etc/dovecot/dovecot.conf': {
+        'content_type': 'mako',
+        'context': {
+            'admin_email': node.metadata.get('mailserver/admin_email'),
+        },
+        'needs': {
+            'pkg_apt:'
+        },
+        'triggers': {
+            'svc_systemd:dovecot:restart',
+        },
+    },
+    '/etc/dovecot/dovecot-sql.conf': {
+        'content_type': 'mako',
+        'context': node.metadata.get('mailserver/database'),
+        'needs': {
+            'pkg_apt:'
+        },
+        'triggers': {
+            'svc_systemd:dovecot:restart',
+        },
+    },
+}
+
+actions = {
+    'dovecot_generate_dhparam': {
+        'command': 'openssl dhparam -out /etc/dovecot/ssl/dhparam.pem 2048',
+        'unless': 'test -f /etc/dovecot/ssl/dhparam.pem',
+        'cascade_skip': False,
+        'needs': {
+            'pkg_apt:'
+        },
+        'triggers': {
+            'svc_systemd:dovecot:restart',
+        },
+    },
+}
+
+svc_systemd = {
+    'dovecot': {
+        'needs': {
+            'action:dovecot_generate_dhparam',
+            'file:/etc/dovecot/dovecot.conf',
+            'file:/etc/dovecot/dovecot-sql.conf',
+        },
+    },
+}
diff --git a/bundles/dovecot/metadata.py b/bundles/dovecot/metadata.py
new file mode 100644
index 0000000..09810fd
--- /dev/null
+++ b/bundles/dovecot/metadata.py
@@ -0,0 +1,25 @@
+from bundlewrap.metadata import atomic
+
+defaults = {
+    'apt': {
+        'packages': {
+            'dovecot-imapd': {},
+            'dovecot-lmtpd': {},
+            'dovecot-managesieved': {},
+            'dovecot-pgsql': {},
+            'dovecot-sieve': {},
+        },
+    },
+    'letsencrypt': {
+        'reload_after': {
+            'dovecot',
+        },
+    },
+    'dovecot': {
+        'database': {
+            'dbname': 'mailserver',
+            'dbuser': 'mailserver',
+        },
+    },
+
+}
diff --git a/bundles/mailserver/metadata.py b/bundles/mailserver/metadata.py
new file mode 100644
index 0000000..ed29b28
--- /dev/null
+++ b/bundles/mailserver/metadata.py
@@ -0,0 +1,33 @@
+database_password = repo.vault.password_for(f'{node.name} db mailserver')
+
+defaults = {
+    'mailserver': {
+        'maildir': '/var/vmail',
+        'database': {
+            'host': '127.0.0.1',
+            'name': 'mailserver',
+            'user': 'mailserver',
+            'password': database_password,
+        },
+    },
+    'zfs': {
+        'datasets': {
+            'tank/vmail': {
+                'mountpoint': '/var/vmail',
+                'compression': 'on',
+            },
+        },
+    },
+    'postgresql': {
+        'roles': {
+            'mailserver': {
+                'password': database_password,
+            },
+        },
+        'databases': {
+            'mailserver': {
+                'owner': 'mailserver',
+            },
+        },
+    },
+}
diff --git a/bundles/postfix/items.py b/bundles/postfix/items.py
index 70f18f2..cba5faa 100644
--- a/bundles/postfix/items.py
+++ b/bundles/postfix/items.py
@@ -1,4 +1,6 @@
 assert node.has_bundle('postgresql')
+assert node.has_bundle('dovecot')
+assert node.has_bundle('letsencrypt')
 
 file_options = {
     'triggers': [
@@ -19,22 +21,22 @@ files = {
     },
     '/etc/postfix/virtual_alias_maps.cf': {
         'content_type': 'mako',
-        'context': node.metadata.get('postfix/database'),
+        'context': node.metadata.get('mailserver/database'),
         **file_options,
     },
     '/etc/postfix/virtual_mailbox_domains.cf': {
         'content_type': 'mako',
-        'context': node.metadata.get('postfix/database'),
+        'context': node.metadata.get('mailserver/database'),
         **file_options,
     },
     '/etc/postfix/virtual_mailbox_maps.cf': {
         'content_type': 'mako',
-        'context': node.metadata.get('postfix/database'),
+        'context': node.metadata.get('mailserver/database'),
         **file_options,
     },
     '/etc/postfix/virtual_redirects.cf': {
         'content_type': 'mako',
-        'context': node.metadata.get('postfix/database'),
+        'context': node.metadata.get('mailserver/database'),
         **file_options,
     },
 }
diff --git a/bundles/postfix/metadata.py b/bundles/postfix/metadata.py
index 8940904..3b610e5 100644
--- a/bundles/postfix/metadata.py
+++ b/bundles/postfix/metadata.py
@@ -1,29 +1,13 @@
-database_password = repo.vault.password_for(f'{node.name} db mailserver')
-
 defaults = {
     'apt': {
         'packages': {
             'postfix': {},
+            'postfix-pgsql': {},
         }
     },
-    'postfix': {
-        'database': {
-            'host': '127.0.0.1',
-            'name': 'mailserver',
-            'user': 'mailserver',
-            'password': database_password,
-        }
-    },
-    'postgresql': {
-        'roles': {
-            'mailserver': {
-                'password': database_password,
-            },
-        },
-        'databases': {
-            'mailserver': {
-                'owner': 'mailserver',
-            },
-        },
+    'letsencrypt': {
+        'reload_after': {
+            'postfix',
+        }, 
     },
 }
diff --git a/bundles/postgresql/items.py b/bundles/postgresql/items.py
index e7101c5..0bffd51 100644
--- a/bundles/postgresql/items.py
+++ b/bundles/postgresql/items.py
@@ -1,8 +1,3 @@
-if node.has_bundle('zfs'):
-    pkg_apt[postgresql]\
-        .setdefault('needs', [])\
-        .append('zfs_dataset:tank/postgresql')
-
 for user, config in node.metadata.get('postgresql/roles').items():
     postgres_roles[user] = {
         'password': config['password'],
diff --git a/bundles/zfs/items.py b/bundles/zfs/items.py
new file mode 100644
index 0000000..3603d0c
--- /dev/null
+++ b/bundles/zfs/items.py
@@ -0,0 +1,41 @@
+from json import dumps
+from bundlewrap.metadata import MetadataJSONEncoder
+
+actions = {
+    'modprobe_zfs': {
+        'command': 'modprobe zfs',
+        'unless': 'lsmod | grep ^zfs',
+        'needs': {
+            'pkg_apt:zfs-dkms',
+        },
+        'needed_by': {
+            'pkg_apt:zfs-zed',
+            'pkg_apt:zfsutils-linux',
+            'zfs_dataset:',
+            'zfs_pool:',
+        },
+        'comment': 'If this fails, do a dist-upgrade, reinstall zfs-dkms, reboot',
+    },
+}
+
+svc_systemd = {
+    'zfs-zed': {
+        'needs': {
+            'pkg_apt:zfs-zed'
+        },
+    },
+}
+
+zfs_datasets = node.metadata.get('zfs/datasets', {})
+zfs_pools = {}
+
+# TRIM
+for name, attrs in node.metadata.get('zfs/pools', {}).items():
+    zfs_pools[name] = attrs
+    actions[f'pool_{name}_enable_trim'] = {
+       'command': f'zpool set autotrim=on {name}',
+       'unless':  f'zpool get autotrim -H -o value {name} | grep -q on',
+       'needs':   [
+           f'zfs_pool:{name}'
+       ]
+    }
diff --git a/bundles/zfs/metadata.py b/bundles/zfs/metadata.py
new file mode 100644
index 0000000..4139f33
--- /dev/null
+++ b/bundles/zfs/metadata.py
@@ -0,0 +1,55 @@
+#import re
+
+defaults = {
+    'apt': {
+        'packages': {
+            'linux-headers-amd64': {
+                'needed_by': {
+                    'pkg_apt:zfs-dkms',
+                },
+            },
+            'zfs-dkms': {
+                'needed_by': {
+                    'pkg_apt:zfs-zed',
+                    'pkg_apt:zfsutils-linux',
+                },
+            },
+            'zfs-zed': {
+                'needed_by': {
+                    'zfs_dataset:',
+                    'zfs_pool:',
+                },
+            },
+            'zfsutils-linux': {
+                'needed_by': {
+                    'pkg_apt:zfs-zed',
+                    'zfs_dataset:',
+                    'zfs_pool:',
+                },
+            },
+            'parted': {
+                'needed_by': {
+                    'zfs_pool:',
+                },
+            },
+        },
+    },
+    'zfs': {
+        'datasets': {},
+        'pools': {},
+    },
+}
+
+@metadata_reactor.provides(
+    'zfs/datasets'
+)
+def dataset_defaults(metadata):
+    return {
+        'zfs': {
+            'datasets': {
+                name: {
+                    'compression': 'lz4',
+                } for name, config in metadata.get('zfs/datasets').items()
+            },
+        },
+    }
diff --git a/groups/applications/mailserver.py b/groups/applications/mailserver.py
index 3a69be0..46acec7 100644
--- a/groups/applications/mailserver.py
+++ b/groups/applications/mailserver.py
@@ -1,5 +1,8 @@
 {
     'bundles': [
+        'dovecot',
+        'letsencrypt',
+        'mailserver',
         'postfix',
         'postgresql',
     ],
diff --git a/nodes/htz.mails.py b/nodes/htz.mails.py
index cdfe608..438ac49 100644
--- a/nodes/htz.mails.py
+++ b/nodes/htz.mails.py
@@ -5,6 +5,9 @@
         'mailserver',
         'webserver',
     ],
+    'bundles': [
+        'zfs',
+    ],
     'metadata': {
         'nginx': {
             'vhosts': {
@@ -24,5 +27,9 @@
                 },
             },
         },
+        'mailserver': {
+            'admin_email': 'postmaster@sublimity.de',
+            'hostname': 'mail.sublimity.de',
+        }, 
     },
 }