ssh: manage hostkeys and global known_hosts

bundles/backup/files/backup_path

@@ -7,7 +7,7 @@ then
   /opt/backup/backup_path_via_zfs "$path"
 elif test -d "$path"
 then
-  /opt/backuo/backup_path_via_rsync "$path"
+  /opt/backup/backup_path_via_rsync "$path"
 else
   echo "UNKNOWN PATH: $path"
   exit 1

bundles/backup/files/backup_path_via_rsync

@@ -5,7 +5,7 @@ set -exu
 path=$1
 uuid=$(jq -r .client_uuid < /etc/backup/config.json)
 server=$(jq -r .server_hostname < /etc/backup/config.json)
-ssh="ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 backup-receiver@$server"
+ssh="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 backup-receiver@$server"
 
 rsync -av --rsync-path="sudo rsync" "$path/" "backup-receiver@$server:/mnt/backups/$uuid$path/"
 $ssh sudo zfs snap "tank/$uuid/fs@auto-backup_$(date +"%Y-%m-%d_%H:%M:%S")" 

bundles/backup/files/backup_path_via_zfs

@@ -5,7 +5,7 @@ set -exu
 path=$1
 uuid=$(jq -r .client_uuid < /etc/backup/config.json)
 server=$(jq -r .server_hostname < /etc/backup/config.json)
-ssh="ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 backup-receiver@$server"
+ssh="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 backup-receiver@$server"
 
 source_dataset=$(zfs list -H -o mountpoint,name | grep -P "^$path\t" | cut -d $'\t' -f 2)
 target_dataset="tank/$uuid/$source_dataset"

bundles/ssh/files/ssh_config

@@ -0,0 +1,5 @@
+Host *
+    #UserKnownHostsFile ~/.ssh/known_hosts ~/.ssh/known_hosts.d/%k
+    SendEnv LANG LC_*
+    HashKnownHosts yes
+    GSSAPIAuthentication yes

bundles/ssh/files/sshd_config

@@ -10,7 +10,7 @@ MaxSessions 255
 PubkeyAuthentication yes
 PasswordAuthentication no
 ChallengeResponseAuthentication no
-AuthorizedKeysFile	.ssh/authorized_keys
+AuthorizedKeysFile .ssh/authorized_keys
 UsePAM yes
 
 AllowUsers ${' '.join(users)}

bundles/ssh/items.py

@@ -1,14 +1,61 @@
 if not node.metadata.get('FIXME_dont_touch_sshd', False):
     # on debian bullseye raspberry images, starting the systemd ssh
     # daemon seems to collide with an existing sysv daemon
-    files['/etc/ssh/sshd_config'] = {
+    directories = {
-        'content_type': 'mako',
+        '/etc/ssh': {
-        'context': {
+            'purge': True,
-            'users': sorted(node.metadata.get('ssh/allow_users')),
+            'mode': '0755',
+        }
+    }
+    
+    files = {
+        '/etc/ssh/moduli': {
+            'content_type': 'any',
+        },
+        '/etc/ssh/ssh_config': {
+            'triggers': [
+                'svc_systemd:ssh:restart'
+            ],
+        },
+        '/etc/ssh/ssh_config': {
+            'content_type': 'mako',
+            'context': {
+            },
+            'triggers': [
+                'svc_systemd:ssh:restart'
+            ],
+        },
+        '/etc/ssh/sshd_config': {
+            'content_type': 'mako',
+            'context': {
+                'users': sorted(node.metadata.get('ssh/allow_users')),
+            },
+            'triggers': [
+                'svc_systemd:ssh:restart'
+            ],
+        },
+        '/etc/ssh/ssh_host_ed25519_key': {
+            'content': node.metadata.get('ssh/host_key/private') + '\n',
+            'mode': '0600',
+            'triggers': [
+                'svc_systemd:ssh:restart'
+            ],
+        },
+        '/etc/ssh/ssh_host_ed25519_key.pub': {
+            'content': node.metadata.get('ssh/host_key/public') + '\n',
+            'mode': '0644',
+            'triggers': [
+                'svc_systemd:ssh:restart'
+            ],
+        },
+        '/etc/ssh/ssh_known_hosts': {
+            'content': '\n'.join(
+                repo.libs.ssh.known_hosts_entry_for(other_node)
+                    for other_node in sorted(repo.nodes)
+                    if other_node != node
+                    and other_node.has_bundle('ssh')
+            ) + '\n',
         },
-        'triggers': [
-            'svc_systemd:ssh:restart'
-        ],
     }
 
     svc_systemd['ssh'] = {

bundles/ssh/metadata.py

@@ -1,3 +1,6 @@
+from base64 import b64decode
+
+
 @metadata_reactor.provides(
     'ssh/allow_users',
 )
@@ -11,3 +14,21 @@ def users(metadata):
             ),
         },
     }
+
+
+@metadata_reactor.provides(
+    'ssh/host_key',
+)
+def host_key(metadata):
+    private, public = repo.libs.ssh.generate_ed25519_key_pair(
+        b64decode(str(repo.vault.random_bytes_as_base64_for(f"HostKey {metadata.get('id')}", length=32)))
+    )
+
+    return {
+        'ssh': {
+            'host_key': {
+                'private': private + '\n',
+                'public': public + f' root@{node.name}',
+            }
+        },
+    }

doc/test_protect.service

@@ -0,0 +1,55 @@
+[Unit]
+Description=TEST
+
+[Service]
+Type=oneshot
+ExecStart=/opt/test
+
+DynamicUser=yes
+UMask=077
+ProtectSystem=strict
+ProtectHome=yes
+PrivateTmp=yes
+PrivateDevices=yes # DevicePolicy=closed
+PrivateNetwork=yes
+IPAddressDeny=any
+PrivateUsers=yes
+ProtectHostname=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+RestrictAddressFamilies=none
+RestrictFileSystems=ext4 tmpfs zfs
+RestrictNamespaces=yes
+LockPersonality=yes
+MemoryDenyWriteExecute=yes
+RestrictRealtime=yes
+RestrictSUIDSGID=yes
+RemoveIPC=yes
+PrivateMounts=yes
+SystemCallFilter=~@swap
+SystemCallFilter=~@resources
+SystemCallFilter=~@reboot
+SystemCallFilter=~@raw-io
+SystemCallFilter=~@privileged
+SystemCallFilter=~@obsolete
+SystemCallFilter=~@mount
+SystemCallFilter=~@module
+SystemCallFilter=~@debug
+SystemCallFilter=~@cpu-emulation
+SystemCallFilter=~@clock
+CapabilityBoundingSet=
+ProtectProc=invisible
+ProcSubset=pid
+NoNewPrivileges=yes
+SystemCallArchitectures=native
+
+ReadOnlyPaths=/
+
+NoExecPaths=/
+ExecPaths=/opt/test /bin/bash /lib
+
+[Install]
+WantedBy=multi-user.target

doc/test_protect_2.service

@@ -0,0 +1,55 @@
+[Unit]
+Description=TEST
+
+[Service]
+Type=oneshot
+ExecStart=/opt/test
+
+# user
+UMask=077
+DynamicUser=yes
+PrivateUsers=yes
+RestrictSUIDSGID=yes
+NoNewPrivileges=yes
+LockPersonality=yes
+RemoveIPC=yes
+
+# fs
+ProtectSystem=strict
+ProtectHome=yes
+PrivateTmp=yes
+PrivateDevices=yes
+PrivateNetwork=yes
+ProtectProc=invisible
+ProcSubset=pid
+PrivateMounts=yes
+RestrictFileSystems=ext4 tmpfs zfs
+
+NoExecPaths=/
+ExecPaths=/opt/test /bin /lib /lib64 /usr
+
+TemporaryFileSystem=/var
+TemporaryFileSystem=/var
+
+# network
+IPAddressDeny=any
+RestrictAddressFamilies=none
+
+# syscall
+SystemCallArchitectures=native
+SystemCallFilter=~@swap ~@resources ~@reboot ~@raw-io ~@privileged ~@obsolete ~@mount ~@module ~@debug ~@cpu-emulation ~@clock
+
+# else
+ProtectHostname=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+RestrictNamespaces=yes
+MemoryDenyWriteExecute=yes
+RestrictRealtime=yes
+CapabilityBoundingSet=
+
+[Install]
+WantedBy=multi-user.target

doc/test_temp.service

@@ -0,0 +1,54 @@
+[Unit]
+Description=TEST
+
+[Service]
+Type=oneshot
+ExecStart=/opt/test
+
+TemporaryFileSystem=/
+
+BindReadOnlyPaths=/opt/test /bin /lib /lib64 /usr
+
+
+
+UMask=077
+ProtectHome=yes
+PrivateTmp=yes
+PrivateDevices=yes
+PrivateNetwork=yes
+IPAddressDeny=any
+ProtectHostname=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+RestrictAddressFamilies=none
+RestrictFileSystems=ext4 tmpfs zfs
+RestrictNamespaces=yes
+LockPersonality=yes
+MemoryDenyWriteExecute=yes
+RestrictRealtime=yes
+RestrictSUIDSGID=yes
+RemoveIPC=yes
+PrivateMounts=yes
+SystemCallFilter=~@swap
+SystemCallFilter=~@resources
+SystemCallFilter=~@reboot
+SystemCallFilter=~@raw-io
+SystemCallFilter=~@privileged
+SystemCallFilter=~@obsolete
+SystemCallFilter=~@mount
+SystemCallFilter=~@module
+SystemCallFilter=~@debug
+SystemCallFilter=~@cpu-emulation
+SystemCallFilter=~@clock
+CapabilityBoundingSet=
+ProtectProc=invisible
+ProcSubset=pid
+NoNewPrivileges=yes
+SystemCallArchitectures=native
+
+
+[Install]
+WantedBy=multi-user.target

libs/ssh.py

@@ -1,6 +1,8 @@
 from base64 import b64decode, b64encode
-from hashlib import sha3_224
+from hashlib import sha3_224, sha1
 from functools import cache
+import hmac
+from ipaddress import ip_interface
 
 from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
 from cryptography.hazmat.primitives.serialization import Encoding, PrivateFormat, PublicFormat, NoEncryption
@@ -46,3 +48,43 @@ def generate_ed25519_key_pair(secret):
     # RETURN
     
     return (deterministic_privatekey, public_key)
+
+
+#https://www.fragmentationneeded.net/2017/10/ssh-hashknownhosts-file-format.html
+# test this:
+# - `ssh-keyscan -H 10.0.0.5`
+# - take the salt from the ssh-ed25519 entry (first field after '|1|')
+# - `bw debug -c 'repo.libs.ssh.known_hosts_entry_for(repo.get_node(<node with hostname 10.0.0.5>), <salt from ssh-keygen>)'`
+@cache
+def known_hosts_entry_for(node, test_salt=None):
+    ips = set()
+
+    for network in node.metadata.get('network').values():
+        if network.get('ipv4', None):
+            ips.add(str(ip_interface(network['ipv4']).ip))
+        if network.get('ipv6', None):
+            ips.add(str(ip_interface(network['ipv6']).ip))
+
+    domains = {
+        domain
+            for domain, records in node.metadata.get('dns').items()
+            for type, values in records.items()
+            if type in {'A', 'AAAA'}
+            and set(values) & ips
+    }
+
+    lines = set()
+
+    for hostname in {node.hostname, *ips, *domains}:
+    
+        if test_salt:
+            salt = b64decode(test_salt)
+        else:
+            salt = sha1(node.metadata.get('id').encode()).digest()
+
+        hash = hmac.new(salt, hostname.encode(), sha1).digest()
+        pubkey = node.metadata.get('ssh/host_key/public')
+        
+        lines.add(f'|1|{b64encode(salt).decode()}|{b64encode(hash).decode()} {" ".join(pubkey.split()[:2])}')
+        
+    return '\n'.join(sorted(lines))

libs/systemd.py

@@ -7,7 +7,7 @@ template = '''
 # ${segment.split('#', 2)[1]}
 %     endif
 [${segment.split('#')[0]}]
-%     for option, value in options.items():
+%     for option, value in sorted(options.items()):
 %         if isinstance(value, dict):
 %             for k, v in value.items():
 ${option}=${k}=${v}
@@ -16,6 +16,7 @@ ${option}=${k}=${v}
 %             for item in sorted(value):
 ${option}=${item}
 %             endfor
+%         elif isinstance(value, type(None)):
 %         else:
 ${option}=${str(value)}
 %         endif
@@ -39,5 +40,53 @@ def segment_order(segment):
 def generate_unitfile(data):
     return Template(template).render(
         data=dict(sorted(data.items(), key=segment_order)),
-        order=order
     ).lstrip()
+
+# wip
+def protection():
+    return {
+        # user
+        'UMask': '077',
+        'DynamicUser': 'yes',
+        'PrivateUsers': 'yes',
+        'RestrictSUIDSGID': 'yes',
+        'NoNewPrivileges': 'yes',
+        'LockPersonality': 'yes',
+        'RemoveIPC': 'yes',
+
+        # fs
+        'ProtectSystem': 'strict',
+        'ProtectHome': 'yes',
+        'PrivateTmp': 'yes',
+        'PrivateDevices': 'yes',
+        'ProtectProc': 'invisible',
+        'ProcSubset': 'pid',
+        'PrivateMounts': 'yes',
+        'RestrictFileSystems': {'ext4', 'tmpfs', 'zfs'},
+
+        'NoExecPaths': {'/'},
+        'ExecPaths': {'/bin', '/sbin', '/lib', '/lib64', '/usr'},
+
+        'TemporaryFileSystem': {'/var'},
+
+        # network
+        'IPAddressDeny': 'any',
+        'PrivateNetwork': 'yes',
+        'RestrictAddressFamilies': 'none',
+
+        # syscall
+        'SystemCallArchitectures': 'native',
+        'SystemCallFilter': '~@swap @resources @reboot @raw-io @privileged @obsolete @mount @module @debug @cpu-emulation @clock',
+
+        # else
+        'ProtectHostname': 'yes',
+        'ProtectClock': 'yes',
+        'ProtectKernelTunables': 'yes',
+        'ProtectKernelModules': 'yes',
+        'ProtectKernelLogs': 'yes',
+        'ProtectControlGroups': 'yes',
+        'RestrictNamespaces': 'yes',
+        'MemoryDenyWriteExecute': 'yes',
+        'RestrictRealtime': 'yes',
+        'CapabilityBoundingSet': '',
+    }