icinga

bundles/icinga2/files/check_by_sshmon

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+UNKNOWN=3
+
+if [ -z "$SSHMON_TEST" ]
+then
+    echo 'check_by_sshmon: Env SSHMON_TEST missing' >&2
+    exit $UNKNOWN
+elif [ -z "$SSHMON_COMMAND" ]
+then
+    echo 'check_by_sshmon: Env SSHMON_COMMAND missing' >&2
+    exit $UNKNOWN
+elif [ -z "$SSHMON_HOST" ]
+then
+    echo 'check_by_sshmon: Env SSHMON_HOST missing' >&2
+    exit $UNKNOWN
+fi
+
+ssh sshmon@"$SSHMON_HOST" "sudo $SSHMON_COMMAND"
+
+exitcode=$?
+
+if [ "$exitcode" = 124 ]
+then
+    echo 'check_by_sshmon: Timeout while running check remotely' >&2
+    exit $UNKNOWN
+elif [ "$exitcode" = 255 ]
+then
+    echo 'check_by_sshmon: SSH error' >&2
+    exit $UNKNOWN
+else
+    exit $exitcode
+fi

bundles/icinga2/files/conf.d/api-users.conf

@@ -0,0 +1,10 @@
+% for name, conf in sorted(users.items()):
+object ApiUser "${name}" {
+  password = "${conf['password']}"
+  permissions = [
+% for permission in conf['permissions']:
+    "${permission}",
+% endfor
+  ]
+}
+% endfor

bundles/icinga2/files/conf.d/app.conf

@@ -0,0 +1 @@
+object IcingaApplication "app" { }

bundles/icinga2/files/conf.d/commands.conf

@@ -0,0 +1,198 @@
+/* Command objects */
+
+/* Notification Commands
+ *
+ * Please check the documentation for all required and
+ * optional parameters.
+ */
+
+
+object CheckCommand "sshmon" {
+    import "ipv4-or-ipv6"
+
+    command = [ "/usr/lib/nagios/plugins/check_by_sshmon" ]
+
+    env.SSHMON_TEST = "1234"
+    env.SSHMON_COMMAND = "$command$"
+    env.SSHMON_HOST = "$address$"
+}
+
+
+object NotificationCommand "mail-host-notification" {
+  command = [ ConfigDir + "/scripts/mail-host-notification.sh" ]
+
+  arguments += {
+    "-4" = "$notification_address$"
+    "-6" = "$notification_address6$"
+    "-b" = "$notification_author$"
+    "-c" = "$notification_comment$"
+    "-d" = {
+      required = true
+      value = "$notification_date$"
+    }
+    "-f" = {
+      value = "$notification_from$"
+      description = "Set from address. Requires GNU mailutils (Debian/Ubuntu) or mailx (RHEL/SUSE)"
+    }
+    "-i" = "$notification_icingaweb2url$"
+    "-l" = {
+      required = true
+      value = "$notification_hostname$"
+    }
+    "-n" = {
+      required = true
+      value = "$notification_hostdisplayname$"
+    }
+    "-o" = {
+      required = true
+      value = "$notification_hostoutput$"
+    }
+    "-r" = {
+      required = true
+      value = "$notification_useremail$"
+    }
+    "-s" = {
+      required = true
+      value = "$notification_hoststate$"
+    }
+    "-t" = {
+      required = true
+      value = "$notification_type$"
+    }
+    "-v" = "$notification_logtosyslog$"
+  }
+
+  vars += {
+    notification_address = "$address$"
+    notification_address6 = "$address6$"
+    notification_author = "$notification.author$"
+    notification_comment = "$notification.comment$"
+    notification_type = "$notification.type$"
+    notification_date = "$icinga.long_date_time$"
+    notification_hostname = "$host.name$"
+    notification_hostdisplayname = "$host.display_name$"
+    notification_hostoutput = "$host.output$"
+    notification_hoststate = "$host.state$"
+    notification_useremail = "$user.email$"
+  }
+}
+
+object NotificationCommand "mail-service-notification" {
+  command = [ ConfigDir + "/scripts/mail-service-notification.sh" ]
+
+  arguments += {
+    "-4" = "$notification_address$"
+    "-6" = "$notification_address6$"
+    "-b" = "$notification_author$"
+    "-c" = "$notification_comment$"
+    "-d" = {
+      required = true
+      value = "$notification_date$"
+    }
+    "-e" = {
+      required = true
+      value = "$notification_servicename$"
+    }
+    "-f" = {
+      value = "$notification_from$"
+      description = "Set from address. Requires GNU mailutils (Debian/Ubuntu) or mailx (RHEL/SUSE)"
+    }
+    "-i" = "$notification_icingaweb2url$"
+    "-l" = {
+      required = true
+      value = "$notification_hostname$"
+    }
+    "-n" = {
+      required = true
+      value = "$notification_hostdisplayname$"
+    }
+    "-o" = {
+      required = true
+      value = "$notification_serviceoutput$"
+    }
+    "-r" = {
+      required = true
+      value = "$notification_useremail$"
+    }
+    "-s" = {
+      required = true
+      value = "$notification_servicestate$"
+    }
+    "-t" = {
+      required = true
+      value = "$notification_type$"
+    }
+    "-u" = {
+      required = true
+      value = "$notification_servicedisplayname$"
+    }
+    "-v" = "$notification_logtosyslog$"
+  }
+
+  vars += {
+    notification_address = "$address$"
+    notification_address6 = "$address6$"
+    notification_author = "$notification.author$"
+    notification_comment = "$notification.comment$"
+    notification_type = "$notification.type$"
+    notification_date = "$icinga.long_date_time$"
+    notification_hostname = "$host.name$"
+    notification_hostdisplayname = "$host.display_name$"
+    notification_servicename = "$service.name$"
+    notification_serviceoutput = "$service.output$"
+    notification_servicestate = "$service.state$"
+    notification_useremail = "$user.email$"
+    notification_servicedisplayname = "$service.display_name$"
+  }
+}
+
+/*
+ * If you prefer to use the notification scripts with environment
+ * variables instead of command line parameters, you can use
+ * the following commands. They have been updated from < 2.7
+ * to support the new notification scripts and should help
+ * with an upgrade.
+ * Remove the comment blocks and comment the notification commands above.
+ */
+
+/*
+
+object NotificationCommand "mail-host-notification" {
+  command = [ ConfigDir + "/scripts/mail-host-notification.sh" ]
+
+  env = {
+    NOTIFICATIONTYPE = "$notification.type$"
+    HOSTDISPLAYNAME = "$host.display_name$"
+    HOSTNAME = "$host.name$"
+    HOSTADDRESS = "$address$"
+    HOSTSTATE = "$host.state$"
+    LONGDATETIME = "$icinga.long_date_time$"
+    HOSTOUTPUT = "$host.output$"
+    NOTIFICATIONAUTHORNAME = "$notification.author$"
+    NOTIFICATIONCOMMENT = "$notification.comment$"
+    HOSTDISPLAYNAME = "$host.display_name$"
+    USEREMAIL = "$user.email$"
+  }
+}
+
+object NotificationCommand "mail-service-notification" {
+  command = [ ConfigDir + "/scripts/mail-service-notification.sh" ]
+
+  env = {
+    NOTIFICATIONTYPE = "$notification.type$"
+    SERVICENAME = "$service.name$"
+    HOSTNAME = "$host.name$"
+    HOSTDISPLAYNAME = "$host.display_name$"
+    HOSTADDRESS = "$address$"
+    SERVICESTATE = "$service.state$"
+    LONGDATETIME = "$icinga.long_date_time$"
+    SERVICEOUTPUT = "$service.output$"
+    NOTIFICATIONAUTHORNAME = "$notification.author$"
+    NOTIFICATIONCOMMENT = "$notification.comment$"
+    HOSTDISPLAYNAME = "$host.display_name$"
+    SERVICEDISPLAYNAME = "$service.display_name$"
+    USEREMAIL = "$user.email$"
+  }
+}
+
+*/

bundles/icinga2/files/conf.d/groups.conf

@@ -0,0 +1,37 @@
+/**
+ * Host group examples.
+ */
+
+object HostGroup "linux-servers" {
+  display_name = "Linux Servers"
+
+  assign where host.vars.os == "Linux"
+}
+
+object HostGroup "windows-servers" {
+  display_name = "Windows Servers"
+
+  assign where host.vars.os == "Windows"
+}
+
+/**
+ * Service group examples.
+ */
+
+object ServiceGroup "ping" {
+  display_name = "Ping Checks"
+
+  assign where match("ping*", service.name)
+}
+
+object ServiceGroup "http" {
+  display_name = "HTTP Checks"
+
+  assign where match("http*", service.check_command)
+}
+
+object ServiceGroup "disk" {
+  display_name = "Disk Checks"
+
+  assign where match("disk*", service.check_command)
+}

bundles/icinga2/files/conf.d/notifications.conf

@@ -0,0 +1,33 @@
+/**
+ * The example notification apply rules.
+ *
+ * Only applied if host/service objects have
+ * the custom variable `notification` defined
+ * and containing `mail` as key.
+ *
+ * Check `hosts.conf` for an example.
+ */
+
+apply Notification "mail-icingaadmin" to Host {
+  import "mail-host-notification"
+  user_groups = host.vars.notification.mail.groups
+  users = host.vars.notification.mail.users
+
+  //interval = 2h
+
+  //vars.notification_logtosyslog = true
+
+  assign where host.vars.notification.mail
+}
+
+apply Notification "mail-icingaadmin" to Service {
+  import "mail-service-notification"
+  user_groups = host.vars.notification.mail.groups
+  users = host.vars.notification.mail.users
+
+  //interval = 2h
+
+  //vars.notification_logtosyslog = true
+
+  assign where host.vars.notification.mail
+}

bundles/icinga2/files/conf.d/templates.conf

@@ -0,0 +1,15 @@
+template Host "generic-host" {
+  max_check_attempts = 3
+  check_interval = 1m
+  retry_interval = 30s
+
+  check_command = "hostalive"
+}
+
+template Service "generic-service" {
+  max_check_attempts = 5
+  check_interval = 1m
+  retry_interval = 30s
+}
+
+template User "generic-user" {}

bundles/icinga2/files/conf.d/timeperiods.conf

@@ -0,0 +1,34 @@
+/**
+ * Sample timeperiods for Icinga 2.
+ * Check the documentation for details.
+ */
+
+object TimePeriod "24x7" {
+  display_name = "Icinga 2 24x7 TimePeriod"
+  ranges = {
+    "monday" 	= "00:00-24:00"
+    "tuesday" 	= "00:00-24:00"
+    "wednesday" = "00:00-24:00"
+    "thursday" 	= "00:00-24:00"
+    "friday" 	= "00:00-24:00"
+    "saturday" 	= "00:00-24:00"
+    "sunday" 	= "00:00-24:00"
+  }
+}
+
+object TimePeriod "9to5" {
+  display_name = "Icinga 2 9to5 TimePeriod"
+  ranges = {
+    "monday" 	= "09:00-17:00"
+    "tuesday" 	= "09:00-17:00"
+    "wednesday" = "09:00-17:00"
+    "thursday" 	= "09:00-17:00"
+    "friday" 	= "09:00-17:00"
+  }
+}
+
+object TimePeriod "never" {
+  display_name = "Icinga 2 never TimePeriod"
+  ranges = {
+  }
+}

bundles/icinga2/files/constants.conf

@@ -0,0 +1,6 @@
+const PluginDir = "/usr/lib/nagios/plugins"
+const ManubulonPluginDir = "/usr/lib/nagios/plugins"
+const PluginContribDir = "/usr/lib/nagios/plugins"
+const NodeName = "${hostname}"
+const ZoneName = NodeName
+const TicketSalt = ""

bundles/icinga2/files/features/api.conf

@@ -0,0 +1 @@
+object ApiListener "api" {}

bundles/icinga2/files/features/checker.conf

@@ -0,0 +1 @@
+object CheckerComponent "checker" { }

bundles/icinga2/files/features/ido-pgsql.conf

@@ -0,0 +1,8 @@
+library "db_ido_pgsql"
+
+object IdoPgsqlConnection "ido-pgsql" {
+  user = "icinga2",
+  password = "${db_password}",
+  host = "localhost",
+  database = "icinga2"
+}

bundles/icinga2/files/features/notification.conf

@@ -0,0 +1 @@
+object NotificationComponent "notification" { }

bundles/icinga2/files/features/syslog.conf

@@ -0,0 +1,3 @@
+object SyslogLogger "syslog" {
+    severity = "warning"
+}

bundles/icinga2/files/hosts.d/host.conf

@@ -0,0 +1,38 @@
+<%!
+    from bundlewrap.utils import Fault
+
+    def render_value(key, value):
+        if isinstance(value, Fault):
+            return render_value(key, value.value)
+        elif isinstance(value, type(None)):
+            return '""'
+        elif isinstance(value, bool):
+            return 'true' if value else 'false'
+        elif isinstance(value, int):
+            return str(value)
+        elif isinstance(value, str):
+            if key.endswith('_interval'):
+                return value
+            else:
+                return f'"{value}"'
+        elif isinstance(value, (list, set)):
+            return '[' + ', '.join(render_value(e) for e in sorted(value)) + ']'
+        else:
+            raise Exception(f"cant process type '{type(value)}' of value '{value}'")
+%>
+
+object Host "${host_name}" {
+  import "generic-host"
+  % for key, value in sorted(host_settings.items()):
+  ${key} = ${render_value(key, value)}
+  % endfor
+}
+
+% for service_name, service_config in sorted(services.items()):
+object Service "${service_name}" {
+  import "generic-service"
+  % for key, value in sorted(service_config.items()):
+  ${key} = ${render_value(key, value)}
+  % endfor
+}
+% endfor

bundles/icinga2/files/icinga2.conf

@@ -0,0 +1,10 @@
+include "constants.conf"
+include "zones.conf"
+
+include <itl>
+include <plugins>
+include <plugins-contrib>
+
+include "features.d/*.conf"
+include_recursive "conf.d"
+include "hosts.d/*.conf"

bundles/icinga2/files/zones.conf

@@ -0,0 +1,7 @@
+object Endpoint NodeName {
+  host = NodeName
+}
+
+object Zone ZoneName {
+  endpoints = [ NodeName ]
+}

bundles/icinga2/items.py

@@ -0,0 +1,259 @@
+from ipaddress import ip_interface
+
+directories = {
+    '/etc/icinga2': {
+       'purge': True,
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0750',
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+    '/etc/icinga2/conf.d': {
+        'purge': True,
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0750',
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+    '/etc/icinga2/hosts.d': {
+        'purge': True,
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0750',
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+    '/etc/icinga2/features.d': {
+        'purge': True,
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0750',
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+    '/etc/icinga2/scripts': {
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0750',
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+    '/var/lib/icinga2': {
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0750',
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+    '/var/lib/icinga2/certs': {
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0700',
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+}
+
+files = {
+    '/etc/icinga2/icinga2.conf': {
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0640',
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+    '/etc/icinga2/constants.conf': {
+        'content_type': 'mako',
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0640',
+        'context': {
+            'hostname': node.metadata.get('icinga2/hostname')
+        },
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+    '/etc/icinga2/zones.conf': {
+        'content_type': 'mako',
+        'context': {
+            'hostname': node.metadata.get('icinga2/hostname')
+        },
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0640',
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+    '/etc/icinga2/conf.d/api-users.conf': {
+        'source': 'conf.d/api-users.conf',
+        'content_type': 'mako',
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0640',
+        'context': {
+            'users': node.metadata.get('icinga2/api_users'),
+        },
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+    '/etc/icinga2/conf.d/app.conf': {
+        'source': 'conf.d/app.conf',
+        'content_type': 'mako',
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0640',
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+    '/etc/icinga2/conf.d/commands.conf': {
+        'source': 'conf.d/commands.conf',
+        'content_type': 'mako',
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0640',
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+    '/etc/icinga2/conf.d/groups.conf': {
+        'source': 'conf.d/groups.conf',
+        'content_type': 'mako',
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0640',
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+    '/etc/icinga2/conf.d/notifications.conf': {
+        'source': 'conf.d/notifications.conf',
+        'content_type': 'mako',
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0640',
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+    '/etc/icinga2/conf.d/templates.conf': {
+        'source': 'conf.d/templates.conf',
+        'content_type': 'mako',
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0640',
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+    '/etc/icinga2/conf.d/timeperiods.conf': {
+        'source': 'conf.d/timeperiods.conf',
+        'content_type': 'mako',
+        'owner': 'nagios',
+        'group': 'nagios',
+        'mode': '0640',
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+    '/etc/icinga2/features.d/ido-pgsql.conf': {
+        'source': 'features/ido-pgsql.conf',
+        'content_type': 'mako',
+        'owner': 'nagios',
+        'group': 'nagios',
+        'context': {
+            'db_password': node.metadata.get('postgresql/roles/icinga2/password')
+        },
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+    '/etc/icinga2/features.d/syslog.conf': {
+        'source': 'features/syslog.conf',
+        'owner': 'nagios',
+        'group': 'nagios',
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+    '/etc/icinga2/features.d/notification.conf': {
+        'source': 'features/notification.conf',
+        'owner': 'nagios',
+        'group': 'nagios',
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+    '/etc/icinga2/features.d/checker.conf': {
+        'source': 'features/checker.conf',
+        'owner': 'nagios',
+        'group': 'nagios',
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+    '/etc/icinga2/features.d/api.conf': {
+        'source': 'features/api.conf',
+        'owner': 'nagios',
+        'group': 'nagios',
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+    '/var/lib/icinga2/certs/ca.crt': {
+        'content_type': 'download',
+        'source': f'https://letsencrypt.org/certs/isrg-root-x1-cross-signed.pem',
+        'owner': 'nagios',
+        'group': 'nagios',
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    },
+    '/usr/lib/nagios/plugins/check_by_sshmon': {
+        'mode': '0755',
+    },
+}
+
+for other_node in repo.nodes:
+    if other_node.dummy:
+        continue
+    elif not other_node.in_group('monitored'):
+        continue
+
+    files[f'/etc/icinga2/hosts.d/{other_node.name}.conf'] = {
+        'content_type': 'mako',
+        'source': 'hosts.d/host.conf',
+        'owner': 'nagios',
+        'context': {
+            'host_name': other_node.name,
+            'host_settings': {
+                'address': str(ip_interface(other_node.metadata.get('network/internal/ipv4', None) or other_node.metadata.get('wireguard/my_ip')).ip),
+            },
+            'services': other_node.metadata.get('monitoring/services'),
+        },
+        'triggers': [
+            'svc_systemd:icinga2.service:restart',
+        ],
+    }
+
+svc_systemd = {
+    'icinga2.service': {
+        'needs': [
+            'pkg_apt:icinga2-ido-pgsql',
+            'svc_systemd:postgresql',
+        ],
+    },
+}

bundles/icinga2/metadata.py

@@ -0,0 +1,73 @@
+from hashlib import sha3_256
+
+defaults = {
+    'apt': {
+        'packages': {
+            'icinga2': {},
+            'icinga2-ido-pgsql': {},
+            'icingacli': {},
+            'monitoring-plugins': {},
+        },
+        'sources': {
+            'deb https://packages.icinga.com/debian icinga-{release} main',
+        },
+    },
+    'icinga2': {
+        'api_users': {
+            'root': {
+                'password': repo.vault.password_for(f'icinga2 api user root on {node.name}'),
+                'permissions': {'*'},
+            }
+        },
+    },
+    'postgresql': {
+        'databases': {
+            'icinga2': {
+                'owner': 'icinga2',
+            },
+        },
+        'roles': {
+            'icinga2': {
+                'password': repo.vault.password_for(f'psql icinga2 on {node.name}'),
+            },
+        },
+    },
+    'users': {
+        'nagios': {
+            'home': '/var/lib/nagios',
+            'shell': '/usr/sbin/nologin',
+        },
+    },
+    'zfs': {
+        'datasets': {
+            'tank/icinga2': {
+                'mountpoint': '/var/lib/icinga2',
+                'needed_by': {
+                    'pkg_apt:icinga2',
+                    'pkg_apt:icingaweb2',
+                    'pkg_apt:icinga2-ido-pgsql',
+                },
+            },
+        },
+    },
+}
+
+
+@metadata_reactor.provides(
+    'nginx/vhosts',
+)
+def letsencrypt(metadata):
+    return {
+        'letsencrypt': {
+            'domains': {
+                metadata.get('icingaweb2/hostname'): {
+                    'reload': {'icinga2'},
+                    'owner': 'nagios',
+                    'group': 'nagios',
+                    'location': '/var/lib/icinga2/certs',
+                    'privkey_name': metadata.get('hostname') + '.key',
+                    'cert_name': metadata.get('hostname') + '.crt',
+                },
+            },
+        },
+    }

bundles/icingaweb2/README.md

@@ -0,0 +1,4 @@
+- apply
+- open /icingaweb2/setup in browser
+- fill in values from metadata
+- apply

bundles/icingaweb2/items.py

@@ -0,0 +1,63 @@
+directories = {
+    '/etc/icingaweb2': {
+#        'purge': True,
+        'owner': 'www-data',
+        'group': 'icingaweb2',
+        'mode': '2770',
+    },
+    '/etc/icingaweb2/enabledModules': {
+#        'purge': True,
+        'owner': 'www-data',
+        'group': 'icingaweb2',
+        'mode': '2770',
+    },
+    '/etc/icingaweb2/modules': {
+#        'purge': True,
+        'owner': 'www-data',
+        'group': 'icingaweb2',
+        'mode': '2770',
+    },
+}
+
+files = {
+    '/etc/icingaweb2/setup.token': {
+        'content': node.metadata.get('icingaweb2/setup_token'),
+        'owner': 'www-data',
+        'group': 'icingaweb2',
+        'mode': '0660',
+    },
+}
+
+symlinks = {
+    '/etc/icingaweb2/enabledModules/monitoring': {
+        'target': '/usr/share/icingaweb2/modules/monitoring',
+        'owner': 'www-data',
+        'group': 'icingaweb2',
+    },
+}
+
+for name in [
+    'authentication.ini',
+    'config.ini',
+    'groups.ini',
+    'resources.ini',
+    'roles.ini',
+]:
+    files[f'/etc/icingaweb2/{name}'] = {
+        'content': repo.libs.ini.dumps(node.metadata.get(f'icingaweb2/{name}')),
+        'owner': 'www-data',
+        'group': 'icingaweb2',
+        'mode': '0660',
+    }
+
+for name in [
+    'config.ini',
+    'backends.ini',
+    'commandtransports.ini',
+]:
+    files[f'/etc/icingaweb2/modules/monitoring/{name}'] = {
+        'content': repo.libs.ini.dumps(node.metadata.get(f'icingaweb2/monitoring/{name}')),
+        'owner': 'www-data',
+        'group': 'icingaweb2',
+        'mode': '0660',
+    }

bundles/icingaweb2/metadata.py

@@ -0,0 +1,171 @@
+from hashlib import sha3_256
+
+defaults = {
+    'apt': {
+        'packages': {
+            'icingaweb2': {},
+            'php-ldap': {},
+            'php-json': {},
+            'php-intl': {},
+            'php-xml': {},
+            'php-gd': {},
+            'php-imagick': {},
+            'php-pgsql': {},
+            'icingaweb2-module-monitoring': {},
+        },
+        'sources': {
+            'deb https://packages.icinga.com/debian icinga-{release} main',
+            'deb https://packages.icinga.com/debian icinga-{release}-snapshots main',
+        },
+    },
+    'icingaweb2': {
+        'authentication.ini': {
+            'icingaweb2': {
+                'backend': 'db',
+                'resource': 'icingaweb2_db',
+            },
+        },
+        'config.ini': {
+            'global': {
+                'show_stacktraces': '1',
+                'show_application_state_messages': '1',
+                'module_path': '/usr/share/icingaweb2/modules',
+                'config_backend': 'db',
+                'config_resource': 'icingaweb2_db',
+            },
+            'logging': {
+                'log': 'syslog',
+                'level': 'INFO',
+                'application': 'icingaweb2',
+                'facility': 'user',
+            },
+        },
+        'groups.ini': {
+            'icingaweb2': {
+                'backend': 'db',
+                'resource': 'icingaweb2_db',
+            },
+        },
+        'resources.ini': {
+            'icingaweb2_db': {
+                'type': 'db',
+                'db': 'pgsql',
+                'host': 'localhost',
+                'port': '5432',
+                'dbname': 'icingaweb2',
+                'username': 'icingaweb2',
+                'password': str(repo.vault.password_for(f'psql icingaweb2 on {node.name}')),
+                'charset': '',
+                'use_ssl': '0',
+            },
+            'icinga_ido': {
+                'type': 'db',
+                'db': 'pgsql',
+                'host': 'localhost',
+                'port': '5432',
+                'dbname': 'icinga2',
+                'username': 'icinga2',
+                'charset': '',
+                'use_ssl': '0',
+            },
+        },
+        'roles.ini': {
+            'Administrators': {
+                'users': 'root',
+                'permissions': '*',
+                'groups': 'Administrators',
+            },
+        },
+        'monitoring': {
+            'config.ini': {
+                'security': {
+                    'protected_customvars': '*pw*,*pass*,community',
+                },
+            },
+            'backends.ini': {
+                'icinga2': {
+                    'type': 'ido',
+                    'resource': 'icinga_ido',
+                },
+            },
+            'commandtransports.ini': {
+                'icinga2': {
+                    'transport': 'api',
+                    'host': 'localhost',
+                    'port': '5665',
+                    'username': 'root',
+                },
+            },
+        },
+    },
+    'postgresql': {
+        'databases': {
+            'icingaweb2': {
+                'owner': 'icingaweb2',
+            },
+        },
+        'roles': {
+            'icingaweb2': {
+                'password': str(repo.vault.password_for(f'psql icingaweb2 on {node.name}')),
+            },
+        },
+    },
+    'redis': {
+        'icingaweb2': {},
+    },
+}
+
+
+@metadata_reactor.provides(
+    'icingaweb2/hostname',
+    'icingaweb2/resources.ini/icinga_ido/icinga2/password',
+    'icingaweb2/monitoring/commandtransports.ini/icinga2/password',
+)
+def stuff(metadata):
+    return {
+        'icingaweb2': {
+            'hostname': metadata.get('icinga2/hostname'),
+            'resources.ini': {
+                'icinga_ido': {
+                    'password': str(metadata.get('postgresql/roles/icinga2/password')),
+                },
+            },
+            'monitoring': {
+                'commandtransports.ini': {
+                    'icinga2': {
+                        'password': str(metadata.get('icinga2/api_users/root/password')),
+                    },
+                },
+            },
+        },
+    }
+
+
+@metadata_reactor.provides(
+    'icingaweb2/setup_token',
+    'icingaweb2/root_password',
+)
+def setup_token(metadata):
+    return {
+        'icingaweb2': {
+            'setup_token': sha3_256(metadata.get('id').encode()).hexdigest()[:16],
+            'root_password': str(repo.vault.password_for(f"icingaweb2 root user on {metadata.get('id')}")),
+        },
+    }
+
+
+@metadata_reactor.provides(
+    'nginx/vhosts',
+)
+def nginx(metadata):
+    return {
+        'nginx': {
+            'vhosts': {
+                metadata.get('icingaweb2/hostname'): {
+                    'content': 'icingaweb2/vhost.conf',
+                    'context': {
+                    },
+                },
+            },
+        },
+    }

bundles/letsencrypt/files/hook.sh

@@ -26,19 +26,23 @@ deploy_cert() {
   CERTFILE="$3"
   FULLCHAINFILE="$4"
   CHAINFILE="$5"
-  
+
   case $DOMAIN in
   % for domain, conf in sorted(domains.items()):
 <%   if not conf: continue %>\
     ${domain})
       % if conf.get('location', None):
-      cat "$KEYFILE" > "${conf['location']}/privkey.pem"
-      cat "$CERTFILE" > "${conf['location']}/cert.pem"
-      cat "$FULLCHAINFILE" > "${conf['location']}/fullchain.pem"
-      cat "$CHAINFILE" > "${conf['location']}/chain.pem"
+      cat "$KEYFILE" > "${conf['location']}/${conf.get('privkey_name', 'privkey.pem')}"
+      cat "$CERTFILE" > "${conf['location']}/${conf.get('cert_name', 'cert.pem')}"
+      cat "$FULLCHAINFILE" > "${conf['location']}/${conf.get('fullchain_name', 'fullchain.pem')}"
+      cat "$CHAINFILE" > "${conf['location']}/${conf.get('chain_name', 'chain.pem')}"
       % endif
       % if conf.get('owner', None):
-      chown ${conf['owner']} "${conf['location']}/privkey.pem" "${conf['location']}/cert.pem" "${conf['location']}/fullchain.pem" "${conf['location']}/chain.pem"
+      chown ${conf['owner']}:${conf.get('group', '')} \
+        "${conf['location']}/${conf.get('privkey_name', 'privkey.pem')}" \
+        "${conf['location']}/${conf.get('cert_name', 'cert.pem')}" \
+        "${conf['location']}/${conf.get('fullchain_name', 'fullchain.pem')}" \
+        "${conf['location']}/${conf.get('chain_name', 'chain.pem')}"
       % endif
       % for service in sorted(conf.get('reload', [])):
       systemctl reload-or-restart ${service}

bundles/monitored/metadata.py

@@ -0,0 +1,55 @@
+defaults = {
+    'monitoring': {
+        'services': {
+            'test': {
+                'vars.command': '/bin/ls /',
+            },
+        },
+    },
+}
+
+
+@metadata_reactor.provides(
+    'monitoring/services',
+)
+def default_check_command(metadata):
+    services = {}
+
+    for name, conf in metadata.get('monitoring/services').items():
+        services[name] = {}
+
+        if 'host_name' not in conf:
+            services[name]['host_name'] = node.name
+
+        if 'check_command' not in conf:
+            services[name]['check_command'] = 'sshmon'
+
+    return {
+        'monitoring': {
+            'services': services,
+        },
+    }
+
+
+
+@metadata_reactor.provides(
+    'users/sshmon/authorized_users',
+    'sudoers/sshmon',
+)
+def user(metadata):
+    return {
+        'users': {
+            'sshmon': {
+                'authorized_users': {
+                    'nagios@' + metadata.get('monitoring/icinga2_node'),
+                }
+            },
+        },
+        'sudoers': {
+            'sshmon': {
+                conf['vars.command']
+                    for conf in metadata.get('monitoring/services').values()
+                    if conf['check_command'] == 'sshmon'
+            },
+        },
+    }

bundles/systemd-timers/files/check_systemd_timer

@@ -0,0 +1,14 @@
+Timer=$1
+
+Triggers=$(systemctl show ${Timer}.timer --property=Triggers --value)
+if systemctl is-failed "$Triggers"
+then
+  InvocationID=$(systemctl show "$Triggers" --property=InvocationID --value)
+  ExitCode=$(systemctl show "$Triggers" -p ExecStartEx --value | sed 's/^{//' | sed 's/}$//' | tr ';' '\n' | xargs -n 1 | grep '^status=' | cut -d '=' -f 2)
+  journalctl INVOCATION_ID="$InvocationID" --output cat # timer invocation output
+  echo "-----------------"
+  journalctl _SYSTEMD_INVOCATION_ID="$InvocationID" --output cat -n 32 # service invocation output
+  exit 1
+else
+  exit 0
+fi

bundles/systemd-timers/items.py

@@ -2,3 +2,7 @@ svc_systemd['cron'] = {
     'enabled': False,
     'running': False,
 }
+
+files['/usr/lib/nagios/plugins/check_systemd_timer'] = {
+    'mode': '0755',
+}

bundles/systemd-timers/metadata.py

@@ -48,3 +48,19 @@ def systemd(metadata):
             'services': services,
         },
     }
+
+
+@metadata_reactor.provides(
+    'monitoring/services',
+)
+def monitoring(metadata):
+    return {
+        'monitoring': {
+            'services': {
+                f'{name}.timer': {
+                    'vars.command': f'/usr/lib/nagios/plugins/check_systemd_timer {name}'
+                }
+                    for name in metadata.get('systemd-timers')
+            },
+        },
+    }

data/apt/keys/packages.icinga.com.asc

@@ -0,0 +1,30 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v2.0.19 (GNU/Linux)
+
+mQGiBFKHzk4RBACSHMIFTtfw4ZsNKAA03Gf5t7ovsKWnS7kcMYleAidypqhOmkGg
+0petiYsMPYT+MOepCJFGNzwQwJhZrdLUxxMSWay4Xj0ArgpD9vbvU+gj8Tb02l+x
+SqNGP8jXMV5UnK4gZsrYGLUPvx47uNNYRIRJAGOPYTvohhnFJiG402dzlwCg4u5I
+1RdFplkp9JM6vNM9VBIAmcED/2jr7UQGsPs8YOiPkskGHLh/zXgO8SvcNAxCLgbp
+BjGcF4Iso/A2TAI/2KGJW6kBW/Paf722ltU6s/6mutdXJppgNAz5nfpEt4uZKZyu
+oSWf77179B2B/Wl1BsX/Oc3chscAgQb2pD/qPF/VYRJU+hvdQkq1zfi6cVsxyREV
+k+IwA/46nXh51CQxE29ayuy1BoIOxezvuXFUXZ8rP6aCh4KaiN9AJoy7pBieCzsq
+d7rPEeGIzBjI+yhEu8p92W6KWzL0xduWfYg9I7a2GTk8CaLX2OCLuwnKd7RVDyyZ
+yzRjWs0T5U7SRAWspLStYxMdKert9lLyQiRHtLwmlgBPqa0gh7Q+SWNpbmdhIE9w
+ZW4gU291cmNlIE1vbml0b3JpbmcgKEJ1aWxkIHNlcnZlcikgPGluZm9AaWNpbmdh
+Lm9yZz6IYAQTEQIAIAUCUofOTgIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJ
+EMbjGcM0QQaCgSQAnRjXdbsyqziqhmxfAKffNJYuMPwdAKCS/IRCVyQzApFBtIBQ
+1xuoym/4C7kCDQRSh85OEAgAvPwjlURCi8z6+7i60no4n16dNcSzd6AT8Kizpv2r
+9BmNBff/GNYGnHyob/DMtmO2esEuVG8w62rO9m1wzzXzjbtmtU7NZ1Tg+C+reU2I
+GNVu3SYtEVK/UTJHAhLcgry9yD99610tYPN2Fx33Efse94mXOreBfCvDsmFGSc7j
+GVNCWXpMR3jTYyGj1igYd5ztOzG63D8gPyOucTTl+RWN/G9EoGBv6sWqk5eCd1Fs
+JlWyQX4BJn3YsCZx3uj1DWL0dAl2zqcn6m1M4oj1ozW47MqM/efKOcV6VvCs9SL8
+F/NFvZcH4LKzeupCQ5jEONqcTlVlnLlIqId95Z4DI4AV9wADBQf/S6sKA4oH49tD
+Yb5xAfUyEp5ben05TzUJbXs0Z7hfRQzy9+vQbWGamWLgg3QRUVPx1e4IT+W5vEm5
+dggNTMEwlLMI7izCPDcD32B5oxNVxlfj428KGllYWCFj+edY+xKTvw/PHnn+drKs
+LE65Gwx4BPHm9EqWHIBX6aPzbgbJZZ06f6jWVBi/N7e/5n8lkxXqS23DBKemapyu
+S1i56sH7mQSMaRZP/iiOroAJemPNxv1IQkykxw2woWMmTLKLMCD/i+4DxejE50tK
+dxaOLTc4HDCsattw/RVJO6fwE414IXHMv330z4HKWJevMQ+CmQGfswvCwgeBP9n8
+PItLjBQAXIhJBBgRAgAJBQJSh85OAhsMAAoJEMbjGcM0QQaCzpAAmwUNoRyySf9p
+5G3/2UD1PMueIwOtAKDVVDXEq5LJPVg4iafNu0SRMwgP0Q==
+=icbY
+-----END PGP PUBLIC KEY BLOCK-----

data/icingaweb2/vhost.conf

@@ -0,0 +1,31 @@
+# icingacli setup config webserver nginx --document-root /usr/share/icingaweb2/public --config /etc/icingaweb2 --fpm-uri 127.0.0.1:9000
+
+server {
+      listen 443      ssl http2;
+      listen [::]:443 ssl http2;
+
+      server_name ${server_name};
+      root /usr/share/icingaweb2/public;
+
+      ssl_certificate /var/lib/dehydrated/certs/${server_name}/fullchain.pem;
+      ssl_certificate_key /var/lib/dehydrated/certs/${server_name}/privkey.pem;
+
+      location / {
+        return 302 /icingaweb2/index.php;
+      }
+
+      location ~ ^/icingaweb2/index\.php(.*)$ {
+        fastcgi_pass unix:/run/php/php7.4-fpm.sock;
+        fastcgi_index index.php;
+        include fastcgi_params;
+        fastcgi_param SCRIPT_FILENAME /usr/share/icingaweb2/public/index.php;
+        fastcgi_param ICINGAWEB_CONFIGDIR /etc/icingaweb2;
+        fastcgi_param REMOTE_USER $remote_user;
+      }
+
+      location ~ ^/icingaweb2(.+)? {
+        alias /usr/share/icingaweb2/public;
+        index index.php;
+        try_files $1 $uri $uri/ /icingaweb2/index.php$is_args$args;
+      }
+}

data/nginx/proxy_pass.conf

@@ -2,7 +2,7 @@ server {
     listen 443 ssl http2;
     listen [::]:443 ssl http2;
     server_name ${server_name};
-    
+
     ssl_certificate /var/lib/dehydrated/certs/${server_name}/fullchain.pem;
     ssl_certificate_key /var/lib/dehydrated/certs/${server_name}/privkey.pem;
 

groups/applications/monitored.py

@@ -1,10 +1,14 @@
 {
     'bundles': [
         'telegraf',
+        'monitored',
     ],
     'metadata': {
         'telegraf': {
             'influxdb_node': 'home.server',
         },
+        'monitoring': {
+            'icinga2_node': 'home.server',
+        },
     },
 }

nodes/home.server.py

@@ -18,7 +18,8 @@
         'gitea',
 #        'gollum',
         'grafana',
-        'grub',
+        'icinga2',
+        'icingaweb2',
         'influxdb2',
         'mirror',
         'postgresql',
@@ -73,6 +74,12 @@
         'grub': {
             'kernel_params': {'nomodeset'}, # nvidia GT1030 freeze fix
         },
+        'icinga2': {
+            'hostname': 'icinga.sublimity.de',
+        },
+        'icingaweb2': {
+            'hostname': 'icinga.sublimity.de',
+        },
         'influxdb': {
             'hostname': 'influxdb.sublimity.de',
             'admin_token': '!decrypt:encrypt$gAAAAABg3z5PcaLYmUpcElJ07s_G-iYwnS8d532TcR8xUYbZfttT-B736zgR6J726mzKAFNYlIfJ7amNLIzi2ETDH5TAXWsOiAKpX8WC_dPBAvG3uXGtcPYENjdeuvllSagZzPt0hCIZQZXg--Z_YvzaX9VzNrVAgGD-sXQnghN5_Vhf9gVxxwP---VB_6iNlsf61Nc4axoS',