wip

bundles/influxdb2/README.md

@@ -0,0 +1,10 @@
+# setup
+
+- apply influxdb to server
+- write client_token into influxdb metadata:
+  `influx auth list --json | jq -r '.[] | select (.description == "client_token") | .token'`
+- apply clients
+
+# reset password
+
+Opening /var/lib/influxdb/influxd.bolt with https://github.com/br0xen/boltbrowser might help

bundles/influxdb2/items.py

@@ -1,2 +1,73 @@
-#sudo systemctl unmask influxdb.service
-#sudo systemctl start influxdb
+from tomlkit import dumps
+from shlex import quote
+
+directories['/var/lib/influxdb'] = {
+    'owner': 'influxdb',
+    'group': 'influxdb',
+    'needs': [
+        'zfs_dataset:tank/influxdb',
+    ],
+}
+
+directories['/etc/influxdb'] = {
+    'purge': True,
+}
+files['/etc/influxdb/config.toml'] = {
+    'content': dumps(node.metadata.get('influxdb/config')),
+    'triggers': [
+        'svc_systemd:influxdb:restart',
+    ]
+}
+
+svc_systemd['influxdb'] = {
+    'needs': [
+        'directory:/var/lib/influxdb',
+        'file:/etc/influxdb/config.toml',
+        'pkg_apt:influxdb2',
+    ]
+}
+
+actions['wait_for_influxdb_start'] = {
+    'command': 'sleep 5',
+    'triggered': True,
+    'triggered_by': [
+        'svc_systemd:influxdb',
+        'svc_systemd:influxdb:restart',
+    ]
+}
+
+actions['setup_influxdb'] = {
+    'command': 'influx setup --username={username} --password={password} --org={org} --bucket={bucket} --token={token} --retention=0 --force'.format(
+        username=node.metadata.get('influxdb/username'),
+        password=quote(str(node.metadata.get('influxdb/password'))),
+        org=node.metadata.get('influxdb/org'),
+        bucket=node.metadata.get('influxdb/bucket'),
+        token=str(node.metadata.get('influxdb/admin_token')),
+    ),
+    'unless': 'influx bucket list',
+    'needs': [
+        'action:wait_for_influxdb_start',
+    ],
+}
+
+files['/root/.influxdbv2/configs'] = {
+    'content': dumps({
+        node.metadata.get('influxdb/bucket'): {
+            'url': f"http://localhost:{node.metadata.get('influxdb/port')}",
+            'token': str(node.metadata.get('influxdb/admin_token')),
+            'org': node.metadata.get('influxdb/org'),
+            'active': True,
+        },
+    }),
+    'needs': [
+        'action:setup_influxdb',
+    ],
+}
+
+actions['create_influxdb_client_token'] = {
+    'command': 'influx auth create --description client_token --write-buckets --read-telegrafs',
+    'unless': """influx auth list --json | jq -r '.[] | select (.description == "client_token") | .token' | wc -l | grep -q ^1$""",
+    'needs': [
+        'file:/root/.influxdbv2/configs',
+    ],
+}

bundles/influxdb2/metadata.py

@@ -1,3 +1,5 @@
+from ipaddress import ip_interface
+
 defaults = {
     'apt': {
         'packages': {
@@ -7,4 +9,58 @@ defaults = {
             'deb https://repos.influxdata.com/debian {release} stable',
         ],
     },
+    'influxdb': {
+        'port': '8200',
+        'username': 'admin',
+        'org': 'default',
+        'bucket': 'default',
+        'config': {
+            'bolt-path': '/var/lib/influxdb/influxd.bolt',
+            'engine-path': '/var/lib/influxdb/engine',
+            'reporting-disabled': True,
+            'http-bind-address': ':8200',
+        },
+    },
+    'zfs': {
+        'datasets': {
+            'tank/influxdb': {
+                'mountpoint': '/var/lib/influxdb'
+            },
+        },
+    },
 }
+
+@metadata_reactor.provides(
+    'influxdb/password',
+)
+def admin_password(metadata):
+    return {
+        'influxdb': {
+            'password': repo.vault.password_for(f"{metadata.get('id')} influxdb admin"),
+            'admin_token': repo.vault.random_bytes_as_base64_for(f"{metadata.get('id')} influxdb default token", length=64),
+        },
+    }
+
+
+@metadata_reactor.provides(
+    'dns',
+)
+def dns(metadata):
+    dns = {}
+    
+    dns[metadata.get('influxdb/hostname')] = {
+        'A': [
+            str(ip_interface(network['ipv4']).ip)
+                for network in metadata.get('network').values()
+                if 'ipv4' in network
+        ],
+        'AAAA': [
+            str(ip_interface(network['ipv6']).ip)
+                for network in metadata.get('network').values()
+                if 'ipv6' in network
+        ],
+    }
+
+    return {
+        'dns': dns,
+    }

bundles/telegraf/items.py

@@ -0,0 +1,14 @@
+from tomlkit import dumps
+
+files['/etc/telegraf/telegraf.conf'] = {
+    'content': dumps(node.metadata.get('telegraf/config')),
+    'triggers': [
+        'svc_systemd:telegraf:restart',
+    ],
+}
+
+svc_systemd['telegraf'] = {
+    'needs': [
+        'file:/etc/telegraf/telegraf.conf',
+    ],
+}

bundles/telegraf/metadata.py

@@ -0,0 +1,77 @@
+defaults = {
+    'apt': {
+        'packages': {
+            'telegraf': {},
+        },
+        'sources': [
+            'deb https://repos.influxdata.com/debian {release} stable',
+        ],
+    },
+    'telegraf': {
+        'config': {
+            'agent': {
+                'hostname': node.name,
+                'collection_jitter': '0s',
+                'flush_interval': '10s',
+                'flush_jitter': '0s',
+                'interval': '10s',
+                'metric_batch_size': 1000,
+                'metric_buffer_limit': 10000,
+                'omit_hostname': False,
+                'round_interval': True
+            },
+            'inputs': {
+                'cpu': [{
+                    'collect_cpu_time': False,
+                    'percpu': True,
+                    'report_active': False,
+                    'totalcpu': True
+                }],
+                'disk': [{
+                    'ignore_fs': [
+                        'tmpfs',
+                        'devtmpfs',
+                        'devfs',
+                        'iso9660',
+                        'overlay',
+                        'aufs',
+                        'squashfs'
+                    ],
+                }],
+                'diskio': [{}],
+                'kernel': [{}],
+                'mem': [{}],
+                'processes': [{}],
+                'swap': [{}],
+                'system': [{}],
+            },
+        },
+    },
+}
+
+
+@metadata_reactor.provides(
+    'telegraf/config/outputs/influxdb_v2',
+)
+def influxdb(metadata):
+    influxdb_node = repo.get_node(metadata.get('telegraf/influxdb_node'))
+
+    influxdb_server_url = "http://{hostname}:{port}".format(
+        hostname=influxdb_node.metadata.get('influxdb/hostname'),
+        port=influxdb_node.metadata.get('influxdb/port'),
+    )
+
+    return {
+        'telegraf': {
+            'config': {
+                'outputs': {
+                    'influxdb_v2': [{
+                        'urls': [influxdb_server_url],
+                        'token': str(influxdb_node.metadata.get(f'influxdb/client_token')),
+                        'organization': influxdb_node.metadata.get('influxdb/org'),
+                        'bucket': influxdb_node.metadata.get('influxdb/bucket'),
+                    }]
+                },
+            },
+        },
+    }

groups/applications/monitored.py

@@ -0,0 +1,10 @@
+{
+    'bundles': [
+        'telegraf',
+    ],
+    'metadata': {
+        'telegraf': {
+            'influxdb_node': 'home.server',
+        },
+    },
+}

nodes.py

@@ -1,9 +1,30 @@
 from os import walk
 from os.path import join, basename, splitext
 
+converters = {
+    '32_random_bytes_as_base64_for': lambda x: vault.random_bytes_as_base64_for(x, length=32),
+    'decrypt': lambda x: vault.decrypt(x),
+    'decrypt_file': lambda x: vault.decrypt_file(x),
+    'password_for': lambda x: vault.password_for(x),
+}
+
+def demagify(data):
+    if isinstance(data, str):
+        for name, converter in converters.items():
+            if data.startswith(f'!{name}:'):
+                return converter(data[len(name) + 2:])
+        else:
+            return data
+    elif isinstance(data, dict):
+        return type(data)({key: demagify(value) for key, value in data.items()})
+    elif isinstance(data, (list, set, tuple)):
+        return type(data)([demagify(element) for element in data])
+    else:
+        return data
+
 for root, dirs, files in walk(join(repo_path, "nodes")):
     for filename in files:
         if filename.endswith(".py"):
             node = join(root, filename)
             with open(node, 'r', encoding='utf-8') as f:
-                nodes[splitext(basename(filename))[0]] = eval(f.read())
+                nodes[splitext(basename(filename))[0]] = demagify(eval(f.read()))

nodes/home.server.py

@@ -8,8 +8,10 @@
     ],
     'bundles': [
         'gitea',
+        'influxdb2',
         'postgresql',
         'wireguard',
+        'zfs',
     ],
     'metadata': {
         'id': 'af96709e-b13f-4965-a588-ef2cd476437a',
@@ -25,6 +27,10 @@
             'sha256': '0d11d87ce60d5d98e22fc52f2c8c6ba2b54b14f9c26c767a46bf102c381ad128',
             'domain': 'git.sublimity.de',
         },
+        'influxdb': {
+            'hostname': 'influxdb.sublimity.de',
+            'client_token': '!decrypt:encrypt$gAAAAABg25z8fEYjuRkhg4XuYMtJsPO5SaqlexuricXPZAzZ51_iQtPe5v7S503hMFdZ7j-XQUP6Q2y3ovbzhouRYeRZy1W020csOOtBcH08X-ya9cCAOCMnJdujg0MVakxPJhNPa5Ip5XsI4Bjb0EcftNDayQWQsZw1vFHBHllD-ALTisoCdbImD6a1iT4NuT57JGydbWGW',
+        },
         'users': {
             'root': {
                 'shell': '/usr/bin/zsh',
@@ -41,5 +47,15 @@
                 },
             },
         },
+        'zfs': {
+            'pools': {
+                'tank': {
+                    'mirrors': [
+                        '/dev/disk/by-partlabel/zfs-data-1',
+                        '/dev/disk/by-partlabel/zfs-data-2',
+                    ],
+                },
+            },
+        },
     },
 }

nodes/htz.mails.py

@@ -6,13 +6,12 @@
         'hetzner-cloud',
         'debian-10',
         'mailserver',
+        'monitored',
         'webserver',
         'dnsserver',
     ],
     'bundles': [
         'wireguard',
-        'nextcloud', #TEMP
-        'influxdb2', #TEMP
         'zfs',
     ],
     'metadata': {
@@ -53,24 +52,24 @@
                 'gateway6': 'fe80::1',
             }
         },
-        'nginx': {
-            'vhosts': {
-                'nextcloud': {
-                    'domain': 'test.ckn.li',
-                    'ssl': 'letsencrypt',
-                    'letsencrypt': {
-                        'active': True,
-                        'force_ssl': False,
-                    },
-                    'proxy': {
-                        '/': {
-                            'target':   'https://mail.sublimity.de:443',
-                            'websocket': True,
-                        },
-                    },
-                },
-            },
-        },
+        # 'nginx': {
+        #     'vhosts': {
+        #         'nextcloud': {
+        #             'domain': 'test.ckn.li',
+        #             'ssl': 'letsencrypt',
+        #             'letsencrypt': {
+        #                 'active': True,
+        #                 'force_ssl': False,
+        #             },
+        #             'proxy': {
+        #                 '/': {
+        #                     'target':   'https://mail.sublimity.de:443',
+        #                     'websocket': True,
+        #                 },
+        #             },
+        #         },
+        #     },
+        # },
         'mailserver': {
             'hostname': 'mail.sublimity.de',
             'admin_email': 'postmaster@sublimity.de',