diff --git a/bundles/grafana/items.py b/bundles/grafana/items.py index 2072428..1622b73 100644 --- a/bundles/grafana/items.py +++ b/bundles/grafana/items.py @@ -1,9 +1,11 @@ assert node.has_bundle('redis') assert node.has_bundle('postgresql') +from mako.template import Template from shlex import quote +from copy import deepcopy import yaml - +import json svc_systemd['grafana-server'] = { 'needs': [ @@ -29,6 +31,11 @@ directories = { '/etc/grafana/provisioning/datasources': { 'purge': True, }, + '/etc/grafana/provisioning/dashboards': { + 'purge': True, + }, + '/var/lib/grafana': {}, + '/var/lib/grafana/dashboards': {}, } files = { @@ -47,4 +54,61 @@ files = { 'svc_systemd:grafana-server:restart', ], }, + '/etc/grafana/provisioning/dashboards/managed.yaml': { + 'content': yaml.dump({ + 'apiVersion': 1, + 'providers': [{ + 'name': 'Default', + 'folder': 'Generated', + 'type': 'file', + 'options': { + 'path': '/var/lib/grafana/dashboards', + }, + }], + }), + 'triggers': [ + 'svc_systemd:grafana-server:restart', + ], + }, } + +# DASHBOARDS + +with open(repo.path.join([f'data/grafana/dashboard.py'])) as file: + dashboard_template = eval(file.read()) +with open(repo.path.join([f'data/grafana/panel.py'])) as file: + panel_template = eval(file.read()) +with open(repo.path.join([f'data/grafana/flux.mako'])) as file: + flux_template = Template(file.read()) + +bucket = repo.get_node(node.metadata.get('grafana/influxdb_node')).metadata.get('influxdb/bucket') + +for dashboard_id, (node_name, panels) in enumerate(node.metadata.get('grafana/dashboards').items(), start=1): + dashboard = deepcopy(dashboard_template) + dashboard['id'] = dashboard_id + + for panel_id, (panel_name, panel_config) in enumerate(panels.items(), start=1): + panel = deepcopy(panel_template) + panel['id'] = panel_id + panel['title'] = panel_name + + for target_name, target_config in panel_config.items(): + panel['targets'].append({ + 'refId': target_name, + 'query': flux_template.render( + bucket=bucket, + host=node_name, + field=target_name, + filters=target_config, + ).strip() + }) + + dashboard['panels'].append(panel) + + files[f'/var/lib/grafana/dashboards/{node_name}.json'] = { + 'content': json.dumps(dashboard, sort_keys=True, indent=4), + 'triggers': [ + 'svc_systemd:grafana-server:restart', + ] + } + diff --git a/bundles/grafana/metadata.py b/bundles/grafana/metadata.py index c4eeca9..7a52a71 100644 --- a/bundles/grafana/metadata.py +++ b/bundles/grafana/metadata.py @@ -1,3 +1,5 @@ +from mako.template import Template + postgres_password = repo.vault.password_for(f'{node.name} postgres role grafana') defaults = { @@ -29,16 +31,8 @@ defaults = { 'allow_signup': False, }, }, - 'panels': { - 'CPU': { - 'usage_user': { - 'filter': { - '_measurement': 'cpu', - 'cpu': 'cpu-total', - }, - }, - }, - }, + 'dashboards': {}, + 'datasources': {}, }, 'postgresql': { 'databases': { @@ -62,6 +56,26 @@ defaults = { } +@metadata_reactor.provides( + 'grafana/dashboards', +) +def dashboards(metadata): + dashboards = {} + + for monitored_node in repo.nodes: + if monitored_node.metadata.get('telegraf/influxdb_node', None) == metadata.get('grafana/influxdb_node'): + for telegraf_input in monitored_node.metadata.get('telegraf/config/inputs'): + with open(repo.path.join([f'data/grafana/panels/{telegraf_input}.py'])) as file: + dashboards.setdefault(monitored_node.name, {})[telegraf_input] = \ + eval(Template(file.read()).render(metadata=monitored_node.metadata)) + + return { + 'grafana': { + 'dashboards': dashboards, + } + } + + @metadata_reactor.provides( 'grafana/datasources', ) diff --git a/bundles/telegraf/metadata.py b/bundles/telegraf/metadata.py index 3cec953..b5ff7a6 100644 --- a/bundles/telegraf/metadata.py +++ b/bundles/telegraf/metadata.py @@ -18,14 +18,14 @@ defaults = { 'metric_batch_size': 1000, 'metric_buffer_limit': 10000, 'omit_hostname': False, - 'round_interval': True + 'round_interval': True, }, 'inputs': { 'cpu': [{ 'collect_cpu_time': False, 'percpu': True, 'report_active': False, - 'totalcpu': True + 'totalcpu': True, }], 'disk': [{ 'ignore_fs': [ @@ -35,7 +35,7 @@ defaults = { 'iso9660', 'overlay', 'aufs', - 'squashfs' + 'squashfs', ], }], 'diskio': [{}], diff --git a/bundles/zfs/metadata.py b/bundles/zfs/metadata.py index e136633..b151a8a 100644 --- a/bundles/zfs/metadata.py +++ b/bundles/zfs/metadata.py @@ -38,6 +38,13 @@ defaults = { }, }, }, + 'telegraf': { + 'config': { + 'inputs': { + 'zfs': [{}], + }, + }, + }, 'zfs': { 'datasets': {}, 'pools': {}, diff --git a/data/grafana/dashboard.py b/data/grafana/dashboard.py new file mode 100644 index 0000000..b335b89 --- /dev/null +++ b/data/grafana/dashboard.py @@ -0,0 +1,64 @@ +{ +# "id": 1, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": True, + "hide": True, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": True, + "gnetId": None, + "graphTooltip": 0, + "iteration": 1625410820978, + "links": [], + "panels": [], + "refresh": False, + "schemaVersion": 30, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": None, + "current": { + "isNone": True, + "selected": False, + "text": "None", + "value": "" + }, + "datasource": None, + "definition": "", + "description": None, + "error": None, + "hide": 0, + "includeAll": False, + "label": None, + "multi": False, + "name": "query0", + "options": [], + "query": "", + "refresh": 1, + "regex": "", + "skipUrlSync": False, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "2021-07-01T19:00:00.201Z", + "to": "2021-07-01T19:03:10.018Z" + }, + "timepicker": {}, + "timezone": "", + "title": "New dashboard Copy", + "uid": "IBPgYBznk", + "version": 15 +} diff --git a/data/grafana/flux.mako b/data/grafana/flux.mako new file mode 100644 index 0000000..ed41df9 --- /dev/null +++ b/data/grafana/flux.mako @@ -0,0 +1,8 @@ +from(bucket: "${bucket}") + |> range(start: v.timeRangeStart, stop: v.timeRangeStop) + |> filter(fn: (r) => r["host"] == "${host}") +% for key, value in filters.items(): + |> filter(fn: (r) => r["_field"] == "${field}") +% endfor + |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false) + |> yield(name: "mean") diff --git a/data/grafana/panel.py b/data/grafana/panel.py new file mode 100644 index 0000000..ef354a5 --- /dev/null +++ b/data/grafana/panel.py @@ -0,0 +1,75 @@ +{ +# 'id': 1, +# 'title': 'TBD', + 'type': 'timeseries', + 'transformations': [], + 'description': '', + 'fieldConfig': { + 'defaults': { + 'custom': { + 'drawStyle': 'line', + 'lineInterpolation': 'smooth', + 'barAlignment': 0, + 'lineWidth': 1, + 'fillOpacity': 40, + 'gradientMode': 'opacity', + 'spanNulls': False, + 'showPoints': 'never', + 'pointSize': 5, + 'stacking': { + 'mode': 'normal', + 'group': 'A' + }, + 'axisPlacement': 'hidden', + 'axisLabel': '', + 'scaleDistribution': { + 'type': 'linear' + }, + 'hideFrom': { + 'tooltip': False, + 'viz': False, + 'legend': False + }, + 'thresholdsStyle': { + 'mode': 'off', + }, + 'lineStyle': { + 'fill': 'solid', + }, + }, + 'color': { + 'mode': 'palette-classic', + }, + 'thresholds': { + 'mode': 'percentage', + 'steps': [ + { + 'color': 'green', + 'value': None, + }, + { + 'color': 'dark-red', + 'value': 80, + }, + ], + }, + 'mappings': [], + 'displayName': '${__field.name}', + 'unit': 'percent', + }, + 'overrides': [], + }, + 'options': { + 'tooltip': { + 'mode': 'none', + }, + 'legend': { + 'displayMode': 'list', + 'placement': 'bottom', + 'calcs': [], + }, + }, + 'targets': [], + 'transparent': True, + 'datasource': None, +} diff --git a/data/grafana/panels/cpu.py b/data/grafana/panels/cpu.py new file mode 100644 index 0000000..9f36d10 --- /dev/null +++ b/data/grafana/panels/cpu.py @@ -0,0 +1,20 @@ +{ + 'usage_system': { + 'filter': { + '_measurement': 'cpu', + 'cpu': 'cpu-total', + }, + }, + 'usage_iowait': { + 'filter': { + '_measurement': 'cpu', + 'cpu': 'cpu-total', + }, + }, + 'usage_user': { + 'filter': { + '_measurement': 'cpu', + 'cpu': 'cpu-total', + }, + }, +} diff --git a/data/grafana/panels/disk.py b/data/grafana/panels/disk.py new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/data/grafana/panels/disk.py @@ -0,0 +1 @@ +{} diff --git a/data/grafana/panels/diskio.py b/data/grafana/panels/diskio.py new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/data/grafana/panels/diskio.py @@ -0,0 +1 @@ +{} diff --git a/data/grafana/panels/kernel.py b/data/grafana/panels/kernel.py new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/data/grafana/panels/kernel.py @@ -0,0 +1 @@ +{} diff --git a/data/grafana/panels/mem.py b/data/grafana/panels/mem.py new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/data/grafana/panels/mem.py @@ -0,0 +1 @@ +{} diff --git a/data/grafana/panels/processes.py b/data/grafana/panels/processes.py new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/data/grafana/panels/processes.py @@ -0,0 +1 @@ +{} diff --git a/data/grafana/panels/swap.py b/data/grafana/panels/swap.py new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/data/grafana/panels/swap.py @@ -0,0 +1 @@ +{} diff --git a/data/grafana/panels/system.py b/data/grafana/panels/system.py new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/data/grafana/panels/system.py @@ -0,0 +1 @@ +{} diff --git a/data/grafana/panels/zfs.py b/data/grafana/panels/zfs.py new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/data/grafana/panels/zfs.py @@ -0,0 +1 @@ +{} diff --git a/libs/grafana.py b/libs/grafana.py new file mode 100644 index 0000000..73dc1ec --- /dev/null +++ b/libs/grafana.py @@ -0,0 +1,29 @@ +from mako.template import Template +from copy import deepcopy + + +def generate_flux(bucket, host, field, data): + return Template(flux_template).render( + bucket=bucket, + host=host, + field=field, + data=data + ).strip() + + +def generate_panel(bucket, host, title, targets, min=None, max=None): + panel = deepcopy(panel_template) + panel['title'] = title + + if min: + panel['fieldConfig']['defaults']['min'] = min + if max: + panel['fieldConfig']['defaults']['max'] = max + + panel['targets'] = [ + { + 'hide': False, + 'refId': field, + 'query': generate_flux(bucket, host, field, data), + } for field, data in targets.items() + ]