From 4b6afb503d79837c17ad4d3d1fae9a84bf9a6321 Mon Sep 17 00:00:00 2001 From: mwiegand Date: Tue, 29 Jun 2021 22:08:29 +0200 Subject: [PATCH] wip --- bundles/influxdb2/items.py | 58 +++++++++++++++++++++++- bundles/influxdb2/metadata.py | 57 +++++++++++++++++++++++ bundles/telegraf/items.py | 14 ++++++ bundles/telegraf/metadata.py | 77 ++++++++++++++++++++++++++++++++ groups/applications/monitored.py | 10 +++++ nodes/home.server.py | 15 +++++++ nodes/htz.mails.py | 39 ++++++++-------- 7 files changed, 248 insertions(+), 22 deletions(-) create mode 100644 bundles/telegraf/items.py create mode 100644 bundles/telegraf/metadata.py create mode 100644 groups/applications/monitored.py diff --git a/bundles/influxdb2/items.py b/bundles/influxdb2/items.py index e2201a7..00c87db 100644 --- a/bundles/influxdb2/items.py +++ b/bundles/influxdb2/items.py @@ -1,2 +1,56 @@ -#sudo systemctl unmask influxdb.service -#sudo systemctl start influxdb +from tomlkit import dumps +from shlex import quote + +directories['/var/lib/influxdb'] = { + 'owner': 'influxdb', + 'group': 'influxdb', + 'needs': [ + 'zfs_dataset:tank/influxdb', + ], +} + +directories['/etc/influxdb'] = { + 'purge': True, +} +files['/etc/influxdb/config.toml'] = { + 'content': dumps(node.metadata.get('influxdb/config')), + 'triggers': [ + 'svc_systemd:influxdb:restart', + ] +} + +svc_systemd['influxdb'] = { + 'needs': [ + 'directory:/var/lib/influxdb', + 'file:/etc/influxdb/config.toml', + 'pkg_apt:influxdb2', + ] +} + +actions['setup_influxdb'] = { + 'command': 'influx setup --username={username} --password={password} --org={org} --bucket={bucket} --token={token} --retention=0 --force'.format( + username=node.metadata.get('influxdb/username'), + password=quote(str(node.metadata.get('influxdb/password'))), + org=node.metadata.get('influxdb/org'), + bucket=node.metadata.get('influxdb/bucket'), + token=str(node.metadata.get('influxdb/token')), + ), + 'unless': 'influx bucket list', + 'needs': [ + 'svc_systemd:influxdb', + ], +} + +files['/root/.influxdbv2/configs'] = { + 'content': dumps({ + node.metadata.get('influxdb/bucket'): { + 'url': f"http://localhost:{node.metadata.get('influxdb/port')}", + 'token': str(node.metadata.get('influxdb/token')), + 'org': node.metadata.get('influxdb/org'), + 'active': True, + }, + }), + 'needs': [ + 'action:setup_influxdb', + ], +} diff --git a/bundles/influxdb2/metadata.py b/bundles/influxdb2/metadata.py index 145dc1a..b0e997a 100644 --- a/bundles/influxdb2/metadata.py +++ b/bundles/influxdb2/metadata.py @@ -1,3 +1,5 @@ +from ipaddress import ip_interface + defaults = { 'apt': { 'packages': { @@ -7,4 +9,59 @@ defaults = { 'deb https://repos.influxdata.com/debian {release} stable', ], }, + 'influxdb': { + 'port': '8200', + 'username': 'admin', + 'org': 'default', + 'org': 'default', + 'bucket': 'default', + 'config': { + 'bolt-path': '/var/lib/influxdb/influxd.bolt', + 'engine-path': '/var/lib/influxdb/engine', + 'reporting-disabled': True, + 'http-bind-address': ':8200' + }, + }, + 'zfs': { + 'datasets': { + 'tank/influxdb': { + 'mountpoint': '/var/lib/influxdb' + }, + }, + }, } + +@metadata_reactor.provides( + 'influxdb/password', +) +def admin_password(metadata): + return { + 'influxdb': { + 'password': repo.vault.password_for(f"{node.metadata.get('id')} influxdb admin"), + 'token': repo.vault.random_bytes_as_base64_for(f"{node.metadata.get('id')} influxdb default token", length=64), + }, + } + + +@metadata_reactor.provides( + 'dns', +) +def dns(metadata): + dns = {} + + dns[metadata.get('influxdb/hostname')] = { + 'A': [ + str(ip_interface(network['ipv4']).ip) + for network in metadata.get('network').values() + if 'ipv4' in network + ], + 'AAAA': [ + str(ip_interface(network['ipv6']).ip) + for network in metadata.get('network').values() + if 'ipv6' in network + ], + } + + return { + 'dns': dns, + } diff --git a/bundles/telegraf/items.py b/bundles/telegraf/items.py new file mode 100644 index 0000000..6036a94 --- /dev/null +++ b/bundles/telegraf/items.py @@ -0,0 +1,14 @@ +from tomlkit import dumps + +files['/etc/telegraf/telegraf.conf'] = { + 'content': dumps(node.metadata.get('telegraf/config')), + 'triggers': [ + 'svc_systemd:telegraf:restart', + ], +} + +svc_systemd['telegraf'] = { + 'needs': [ + 'file:/etc/telegraf/telegraf.conf', + ], +} diff --git a/bundles/telegraf/metadata.py b/bundles/telegraf/metadata.py new file mode 100644 index 0000000..31b75de --- /dev/null +++ b/bundles/telegraf/metadata.py @@ -0,0 +1,77 @@ +defaults = { + 'apt': { + 'packages': { + 'telegraf': {}, + }, + 'sources': [ + 'deb https://repos.influxdata.com/debian {release} stable', + ], + }, + 'telegraf': { + 'config': { + 'agent': { + 'hostname': node.name, + 'collection_jitter': '0s', + 'flush_interval': '10s', + 'flush_jitter': '0s', + 'interval': '10s', + 'metric_batch_size': 1000, + 'metric_buffer_limit': 10000, + 'omit_hostname': False, + 'round_interval': True + }, + 'inputs': { + 'cpu': [{ + 'collect_cpu_time': False, + 'percpu': True, + 'report_active': False, + 'totalcpu': True + }], + 'disk': [{ + 'ignore_fs': [ + 'tmpfs', + 'devtmpfs', + 'devfs', + 'iso9660', + 'overlay', + 'aufs', + 'squashfs' + ], + }], + 'diskio': [{}], + 'kernel': [{}], + 'mem': [{}], + 'processes': [{}], + 'swap': [{}], + 'system': [{}], + }, + }, + }, +} + + +@metadata_reactor.provides( + 'telegraf/config/outputs/influxdb_v2', +) +def influxdb(metadata): + influxdb_node = repo.get_node(metadata.get('telegraf/influxdb_node')) + + influxdb_server_url = "http://{hostname}:{port}".format( + hostname=influxdb_node.metadata.get('influxdb/hostname'), + port=influxdb_node.metadata.get('influxdb/port'), + ) + + return { + 'telegraf': { + 'config': { + 'outputs': { + 'influxdb_v2': [{ + 'urls': [influxdb_server_url], + 'token': str(influxdb_node.metadata.get('influxdb/token')), + 'organization': influxdb_node.metadata.get('influxdb/org'), + 'bucket': influxdb_node.metadata.get('influxdb/bucket'), + }] + }, + }, + }, + } diff --git a/groups/applications/monitored.py b/groups/applications/monitored.py new file mode 100644 index 0000000..ccf4ee8 --- /dev/null +++ b/groups/applications/monitored.py @@ -0,0 +1,10 @@ +{ + 'bundles': [ + 'telegraf', + ], + 'metadata': { + 'telegraf': { + 'influxdb_node': 'home.server', + }, + }, +} diff --git a/nodes/home.server.py b/nodes/home.server.py index b35b73d..6792841 100644 --- a/nodes/home.server.py +++ b/nodes/home.server.py @@ -8,8 +8,10 @@ ], 'bundles': [ 'gitea', + 'influxdb2', 'postgresql', 'wireguard', + 'zfs', ], 'metadata': { 'id': 'af96709e-b13f-4965-a588-ef2cd476437a', @@ -25,6 +27,9 @@ 'sha256': '0d11d87ce60d5d98e22fc52f2c8c6ba2b54b14f9c26c767a46bf102c381ad128', 'domain': 'git.sublimity.de', }, + 'influxdb': { + 'hostname': 'influxdb.sublimity.de', + }, 'users': { 'root': { 'shell': '/usr/bin/zsh', @@ -41,5 +46,15 @@ }, }, }, + 'zfs': { + 'pools': { + 'tank': { + 'mirrors': [ + '/dev/disk/by-partlabel/zfs-data-1', + '/dev/disk/by-partlabel/zfs-data-2', + ], + }, + }, + }, }, } diff --git a/nodes/htz.mails.py b/nodes/htz.mails.py index 06303f3..0717dce 100644 --- a/nodes/htz.mails.py +++ b/nodes/htz.mails.py @@ -6,13 +6,12 @@ 'hetzner-cloud', 'debian-10', 'mailserver', + 'monitored', 'webserver', 'dnsserver', ], 'bundles': [ 'wireguard', - 'nextcloud', #TEMP - 'influxdb2', #TEMP 'zfs', ], 'metadata': { @@ -53,24 +52,24 @@ 'gateway6': 'fe80::1', } }, - 'nginx': { - 'vhosts': { - 'nextcloud': { - 'domain': 'test.ckn.li', - 'ssl': 'letsencrypt', - 'letsencrypt': { - 'active': True, - 'force_ssl': False, - }, - 'proxy': { - '/': { - 'target': 'https://mail.sublimity.de:443', - 'websocket': True, - }, - }, - }, - }, - }, + # 'nginx': { + # 'vhosts': { + # 'nextcloud': { + # 'domain': 'test.ckn.li', + # 'ssl': 'letsencrypt', + # 'letsencrypt': { + # 'active': True, + # 'force_ssl': False, + # }, + # 'proxy': { + # '/': { + # 'target': 'https://mail.sublimity.de:443', + # 'websocket': True, + # }, + # }, + # }, + # }, + # }, 'mailserver': { 'hostname': 'mail.sublimity.de', 'admin_email': 'postmaster@sublimity.de',