From 82d76f776b89fda33548c81fe139f9938e987a32 Mon Sep 17 00:00:00 2001 From: mwiegand Date: Fri, 2 Jul 2021 18:59:23 +0200 Subject: [PATCH] wip --- bundles/backup-server/metadata.py | 16 ++------ bundles/bind/metadata.py | 19 ++------- bundles/grafana/README.md | 14 +++++++ bundles/grafana/items.py | 41 ++++++++++++++++---- bundles/grafana/metadata.py | 64 ++++++++++++++++++++++++++++++- bundles/influxdb2/README.md | 19 +++++++-- bundles/influxdb2/items.py | 18 +++++---- bundles/influxdb2/metadata.py | 19 ++------- bundles/telegraf/metadata.py | 15 +++----- libs/dns.py | 13 +++++++ nodes/home.server.py | 5 ++- nodes/htz.mails.py | 2 +- requirements.txt | 1 + 13 files changed, 170 insertions(+), 76 deletions(-) create mode 100644 bundles/grafana/README.md create mode 100644 libs/dns.py diff --git a/bundles/backup-server/metadata.py b/bundles/backup-server/metadata.py index 386cb3f..d67be53 100644 --- a/bundles/backup-server/metadata.py +++ b/bundles/backup-server/metadata.py @@ -48,22 +48,12 @@ def zfs(metadata): def dns(metadata): return { 'dns': { - metadata.get('backup-server/hostname'): { - 'A': [ - str(ip_interface(network['ipv4']).ip) - for network in metadata.get('network').values() - if 'ipv4' in network - ], - 'AAAA': [ - str(ip_interface(network['ipv6']).ip) - for network in metadata.get('network').values() - if 'ipv6' in network - ], - }, - }, + metadata.get('backup-server/hostname'): repo.libs.dns.get_a_records(metadata), + } } + @metadata_reactor.provides( 'users/backup-receiver/authorized_keys' ) diff --git a/bundles/bind/metadata.py b/bundles/bind/metadata.py index ae32b71..eb14e12 100644 --- a/bundles/bind/metadata.py +++ b/bundles/bind/metadata.py @@ -14,24 +14,13 @@ defaults = { @metadata_reactor.provides( - 'bind/zones', + 'dns', ) def dns(metadata): return { 'dns': { - metadata.get('bind/domain'): { - 'A': [ - str(ip_interface(network['ipv4']).ip) - for network in metadata.get('network').values() - if 'ipv4' in network - ], - 'AAAA': [ - str(ip_interface(network['ipv6']).ip) - for network in metadata.get('network').values() - if 'ipv6' in network - ] - }, - }, + metadata.get('bind/hostname'): repo.libs.dns.get_a_records(metadata), + } } @@ -80,7 +69,7 @@ def ns_records(metadata): 'bind': { 'zones': { zone: [ - {'name': '@', 'type': 'NS', 'value': f"{metadata.get('bind/domain')}."}, + {'name': '@', 'type': 'NS', 'value': f"{metadata.get('bind/hostname')}."}, ] for zone in metadata.get('bind/zones').keys() }, }, diff --git a/bundles/grafana/README.md b/bundles/grafana/README.md new file mode 100644 index 0000000..61e9ffc --- /dev/null +++ b/bundles/grafana/README.md @@ -0,0 +1,14 @@ +# metadata + +```python +{ + 'hostname': 'example.com', + 'influxdb_node': 'htz.influx', +} +``` + +# links + + +https://github.com/grafana/influxdb-flux-datasource/issues/42 +https://community.grafana.com/t/no-alias-by-when-using-flux/15575/6 diff --git a/bundles/grafana/items.py b/bundles/grafana/items.py index 44e8ccf..2072428 100644 --- a/bundles/grafana/items.py +++ b/bundles/grafana/items.py @@ -2,24 +2,49 @@ assert node.has_bundle('redis') assert node.has_bundle('postgresql') from shlex import quote +import yaml -files['/etc/grafana/grafana.ini'] = { - 'content': repo.libs.ini.dumps(node.metadata.get('grafana/config')), - 'triggers': [ - 'svc_systemd:grafana-server:restart', - ] -} - svc_systemd['grafana-server'] = { 'needs': [ 'pkg_apt:grafana', ], } +admin_password = node.metadata.get('grafana/config/security/admin_password') +port = node.metadata.get('grafana/config/server/http_port') actions['reset_grafana_admin_password'] = { - 'command': f"grafana-cli admin reset-admin-password {quote(node.metadata.get('grafana/config/security/admin_password'))}", + 'command': f"grafana-cli admin reset-admin-password {quote(admin_password)}", + 'unless': f"curl http://admin:{quote(admin_password)}@localhost:{port}/api/org", 'needs': [ 'svc_systemd:grafana-server', ], } + +directories = { + '/etc/grafana': { + }, + '/etc/grafana/provisioning': { + }, + '/etc/grafana/provisioning/datasources': { + 'purge': True, + }, +} + +files = { + '/etc/grafana/grafana.ini': { + 'content': repo.libs.ini.dumps(node.metadata.get('grafana/config')), + 'triggers': [ + 'svc_systemd:grafana-server:restart', + ], + }, + '/etc/grafana/provisioning/datasources/managed.yaml': { + 'content': yaml.dump({ + 'apiVersion': 1, + 'datasources': list(node.metadata.get('grafana/datasources').values()), + }), + 'triggers': [ + 'svc_systemd:grafana-server:restart', + ], + }, +} diff --git a/bundles/grafana/metadata.py b/bundles/grafana/metadata.py index f307904..c4eeca9 100644 --- a/bundles/grafana/metadata.py +++ b/bundles/grafana/metadata.py @@ -12,7 +12,7 @@ defaults = { 'grafana': { 'config': { 'server': { - 'http_port': 3370, + 'http_port': 8300, }, 'database': { 'url': f'postgres://grafana:{postgres_password}@localhost:5432/grafana', @@ -29,6 +29,16 @@ defaults = { 'allow_signup': False, }, }, + 'panels': { + 'CPU': { + 'usage_user': { + 'filter': { + '_measurement': 'cpu', + 'cpu': 'cpu-total', + }, + }, + }, + }, }, 'postgresql': { 'databases': { @@ -50,3 +60,55 @@ defaults = { }, }, } + + +@metadata_reactor.provides( + 'grafana/datasources', +) +def influxdb2(metadata): + influxdb_metadata = repo.get_node(metadata.get('grafana/influxdb_node')).metadata.get('influxdb') + + return { + 'grafana': { + 'datasources': { + f"influxdb@{influxdb_metadata['hostname']}": { + 'type': 'influxdb', + 'url': f"http://{influxdb_metadata['hostname']}:{influxdb_metadata['port']}", + 'jsonData': { + 'version': 'Flux', + 'organization': influxdb_metadata['org'], + 'defaultBucket': influxdb_metadata['bucket'], + }, + 'secureJsonData': { + 'token': str(influxdb_metadata['readonly_token']), + }, + 'editable': False, + 'isDefault': True, + }, + }, + }, + } + + +@metadata_reactor.provides( + 'grafana/datasources', +) +def datasource_key_to_name(metadata): + return { + 'grafana': { + 'datasources': { + name: {'name': name} for name in metadata.get('grafana/datasources').keys() + }, + }, + } + + +@metadata_reactor.provides( + 'dns', +) +def dns(metadata): + return { + 'dns': { + metadata.get('grafana/hostname'): repo.libs.dns.get_a_records(metadata), + } + } diff --git a/bundles/influxdb2/README.md b/bundles/influxdb2/README.md index b52283d..ac2b112 100644 --- a/bundles/influxdb2/README.md +++ b/bundles/influxdb2/README.md @@ -1,9 +1,20 @@ # setup -- apply influxdb to server -- write client_token into influxdb metadata: - `influx auth list --json | jq -r '.[] | select (.description == "client_token") | .token'` -- apply clients +1. apply influxdb to server +2. write `admin`, `readonly` and `writeonly` token into influxdb metadata: + `influx auth list --json | jq -r '.[] | select (.description == "NAME") | .token'` +3. apply clients + +# metadata + +```python +{ + 'hostname': 'example.com', + 'admin_token': 'Wawbd5n...HJS76ez', + 'readonly_token': '5v235b3...6wbnuzz', + 'writeonly_token': '8w4cnos...fn849zg', +} +``` # reset password diff --git a/bundles/influxdb2/items.py b/bundles/influxdb2/items.py index e0edb21..525a67c 100644 --- a/bundles/influxdb2/items.py +++ b/bundles/influxdb2/items.py @@ -64,10 +64,14 @@ files['/root/.influxdbv2/configs'] = { ], } -actions['create_influxdb_client_token'] = { - 'command': 'influx auth create --description client_token --write-buckets --read-telegrafs', - 'unless': """influx auth list --json | jq -r '.[] | select (.description == "client_token") | .token' | wc -l | grep -q ^1$""", - 'needs': [ - 'file:/root/.influxdbv2/configs', - ], -} +for description, permissions in { + 'readonly': '--read-buckets', + 'writeonly': '--write-buckets --read-telegrafs', +}.items(): + actions[f'influxdb_{description}_token'] = { + 'command': f'influx auth create --description {description} {permissions}', + 'unless': f'''influx auth list --json | jq -r '.[] | select (.description == "{description}") | .token' | wc -l | grep -q ^1$''', + 'needs': [ + 'file:/root/.influxdbv2/configs', + ], + } diff --git a/bundles/influxdb2/metadata.py b/bundles/influxdb2/metadata.py index 75e5d61..be179e0 100644 --- a/bundles/influxdb2/metadata.py +++ b/bundles/influxdb2/metadata.py @@ -46,21 +46,8 @@ def admin_password(metadata): 'dns', ) def dns(metadata): - dns = {} - - dns[metadata.get('influxdb/hostname')] = { - 'A': [ - str(ip_interface(network['ipv4']).ip) - for network in metadata.get('network').values() - if 'ipv4' in network - ], - 'AAAA': [ - str(ip_interface(network['ipv6']).ip) - for network in metadata.get('network').values() - if 'ipv6' in network - ], - } - return { - 'dns': dns, + 'dns': { + metadata.get('influxdb/hostname'): repo.libs.dns.get_a_records(metadata), + } } diff --git a/bundles/telegraf/metadata.py b/bundles/telegraf/metadata.py index 6f8b0db..3cec953 100644 --- a/bundles/telegraf/metadata.py +++ b/bundles/telegraf/metadata.py @@ -54,22 +54,17 @@ defaults = { 'telegraf/config/outputs/influxdb_v2', ) def influxdb(metadata): - influxdb_node = repo.get_node(metadata.get('telegraf/influxdb_node')) - - influxdb_server_url = "http://{hostname}:{port}".format( - hostname=influxdb_node.metadata.get('influxdb/hostname'), - port=influxdb_node.metadata.get('influxdb/port'), - ) + influxdb_metadata = repo.get_node(metadata.get('telegraf/influxdb_node')).metadata.get('influxdb') return { 'telegraf': { 'config': { 'outputs': { 'influxdb_v2': [{ - 'urls': [influxdb_server_url], - 'token': str(influxdb_node.metadata.get(f'influxdb/client_token')), - 'organization': influxdb_node.metadata.get('influxdb/org'), - 'bucket': influxdb_node.metadata.get('influxdb/bucket'), + 'urls': [f"http://{influxdb_metadata['hostname']}:{influxdb_metadata['port']}"], + 'token': str(influxdb_metadata['writeonly_token']), + 'organization': influxdb_metadata['org'], + 'bucket': influxdb_metadata['bucket'], }] }, }, diff --git a/libs/dns.py b/libs/dns.py new file mode 100644 index 0000000..f761f9f --- /dev/null +++ b/libs/dns.py @@ -0,0 +1,13 @@ +def get_a_records(metadata): + return { + 'A': [ + str(ip_interface(network['ipv4']).ip) + for network in metadata.get('network').values() + if 'ipv4' in network + ], + 'AAAA': [ + str(ip_interface(network['ipv6']).ip) + for network in metadata.get('network').values() + if 'ipv6' in network + ], + } diff --git a/nodes/home.server.py b/nodes/home.server.py index 28dd95c..8391adc 100644 --- a/nodes/home.server.py +++ b/nodes/home.server.py @@ -31,10 +31,13 @@ }, 'grafana': { 'hostname': 'grafana.sublimity.de', + 'influxdb_node': 'home.server', }, 'influxdb': { 'hostname': 'influxdb.sublimity.de', - 'client_token': '!decrypt:encrypt$gAAAAABg25z8fEYjuRkhg4XuYMtJsPO5SaqlexuricXPZAzZ51_iQtPe5v7S503hMFdZ7j-XQUP6Q2y3ovbzhouRYeRZy1W020csOOtBcH08X-ya9cCAOCMnJdujg0MVakxPJhNPa5Ip5XsI4Bjb0EcftNDayQWQsZw1vFHBHllD-ALTisoCdbImD6a1iT4NuT57JGydbWGW', + 'admin_token': '!decrypt:encrypt$gAAAAABg3z5PcaLYmUpcElJ07s_G-iYwnS8d532TcR8xUYbZfttT-B736zgR6J726mzKAFNYlIfJ7amNLIzi2ETDH5TAXWsOiAKpX8WC_dPBAvG3uXGtcPYENjdeuvllSagZzPt0hCIZQZXg--Z_YvzaX9VzNrVAgGD-sXQnghN5_Vhf9gVxxwP---VB_6iNlsf61Nc4axoS', + 'readonly_token': '!decrypt:encrypt$gAAAAABg3z1-0hnUdzsfivocxhJm58YnPLn96OUvnHiPaehdRhKd6TZBgEPc5YyR07t2-GEUfOvEwoie-O6QsVhWYxrwxNTBXux_iUSx7W6e-fLQA_3MgWf5G97q_3kx_wCgQ6V0iKRyxH988TpNSMACfS4WhCXdSes1CaMpic4VV3S3ox_gCrSHxO7yVXQkJDnOW0MixY5T', + 'writeonly_token': '!decrypt:encrypt$gAAAAABg3z6fGrOy2tNdo03RoYAXmpJoJYkfhBfpblPh_wxYfqmdjtABaD7XyV9mSh9xl8oWQlTAtCk9KndVCDQy7BJ-ju7S3HCKJ0k244Y5YKxUnQtqt9fc9nnm8XD-NOJqLKyfy0QhL_I8dFT02pygoJeCUR5NkZcTKf6julb-iGXI6vWcQgolJTYrW643pHObd-Z-vIEl', }, 'users': { 'root': { diff --git a/nodes/htz.mails.py b/nodes/htz.mails.py index c1038d4..593951d 100644 --- a/nodes/htz.mails.py +++ b/nodes/htz.mails.py @@ -23,7 +23,7 @@ }, 'id': 'ea29bdf0-0b47-4bf4-8346-67d60c9dc4ae', 'bind': { - 'domain': 'ns.sublimity.de', + 'hostname': 'ns.sublimity.de', 'zones': { 'sublimity.de': [], 'freibrief.net': [], diff --git a/requirements.txt b/requirements.txt index c0b176b..aa688ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ bundlewrap>=4.8.0 pycryptodome PyNaCl +PyYAML