From a545a74242844f160e3e3def34a7a76690d6aa80 Mon Sep 17 00:00:00 2001 From: mwiegand Date: Wed, 31 Aug 2022 12:02:40 +0200 Subject: [PATCH] icinga --- bundles/icinga2/files/check_by_sshmon | 33 +++ bundles/icinga2/files/conf.d/api-users.conf | 10 + bundles/icinga2/files/conf.d/app.conf | 1 + bundles/icinga2/files/conf.d/commands.conf | 198 +++++++++++++ bundles/icinga2/files/conf.d/groups.conf | 37 +++ .../icinga2/files/conf.d/notifications.conf | 33 +++ bundles/icinga2/files/conf.d/templates.conf | 15 + bundles/icinga2/files/conf.d/timeperiods.conf | 34 +++ bundles/icinga2/files/constants.conf | 6 + bundles/icinga2/files/features/api.conf | 1 + bundles/icinga2/files/features/checker.conf | 1 + bundles/icinga2/files/features/ido-pgsql.conf | 8 + .../icinga2/files/features/notification.conf | 1 + bundles/icinga2/files/features/syslog.conf | 3 + bundles/icinga2/files/hosts.d/host.conf | 38 +++ bundles/icinga2/files/icinga2.conf | 10 + bundles/icinga2/files/zones.conf | 7 + bundles/icinga2/items.py | 259 ++++++++++++++++++ bundles/icinga2/metadata.py | 73 +++++ bundles/icingaweb2/README.md | 4 + bundles/icingaweb2/items.py | 63 +++++ bundles/icingaweb2/metadata.py | 171 ++++++++++++ bundles/letsencrypt/files/hook.sh | 16 +- bundles/monitored/metadata.py | 55 ++++ .../systemd-timers/files/check_systemd_timer | 14 + bundles/systemd-timers/items.py | 4 + bundles/systemd-timers/metadata.py | 16 ++ data/apt/keys/packages.icinga.com.asc | 30 ++ data/icingaweb2/vhost.conf | 31 +++ data/nginx/proxy_pass.conf | 2 +- groups/applications/monitored.py | 4 + nodes/home.server.py | 9 +- 32 files changed, 1179 insertions(+), 8 deletions(-) create mode 100644 bundles/icinga2/files/check_by_sshmon create mode 100644 bundles/icinga2/files/conf.d/api-users.conf create mode 100644 bundles/icinga2/files/conf.d/app.conf create mode 100644 bundles/icinga2/files/conf.d/commands.conf create mode 100644 bundles/icinga2/files/conf.d/groups.conf create mode 100644 bundles/icinga2/files/conf.d/notifications.conf create mode 100644 bundles/icinga2/files/conf.d/templates.conf create mode 100644 bundles/icinga2/files/conf.d/timeperiods.conf create mode 100644 bundles/icinga2/files/constants.conf create mode 100644 bundles/icinga2/files/features/api.conf create mode 100644 bundles/icinga2/files/features/checker.conf create mode 100644 bundles/icinga2/files/features/ido-pgsql.conf create mode 100644 bundles/icinga2/files/features/notification.conf create mode 100644 bundles/icinga2/files/features/syslog.conf create mode 100644 bundles/icinga2/files/hosts.d/host.conf create mode 100644 bundles/icinga2/files/icinga2.conf create mode 100644 bundles/icinga2/files/zones.conf create mode 100644 bundles/icinga2/items.py create mode 100644 bundles/icinga2/metadata.py create mode 100644 bundles/icingaweb2/README.md create mode 100644 bundles/icingaweb2/items.py create mode 100644 bundles/icingaweb2/metadata.py create mode 100644 bundles/monitored/metadata.py create mode 100644 bundles/systemd-timers/files/check_systemd_timer create mode 100644 data/apt/keys/packages.icinga.com.asc create mode 100644 data/icingaweb2/vhost.conf diff --git a/bundles/icinga2/files/check_by_sshmon b/bundles/icinga2/files/check_by_sshmon new file mode 100644 index 0000000..c9a31df --- /dev/null +++ b/bundles/icinga2/files/check_by_sshmon @@ -0,0 +1,33 @@ +#!/bin/sh + +UNKNOWN=3 + +if [ -z "$SSHMON_TEST" ] +then + echo 'check_by_sshmon: Env SSHMON_TEST missing' >&2 + exit $UNKNOWN +elif [ -z "$SSHMON_COMMAND" ] +then + echo 'check_by_sshmon: Env SSHMON_COMMAND missing' >&2 + exit $UNKNOWN +elif [ -z "$SSHMON_HOST" ] +then + echo 'check_by_sshmon: Env SSHMON_HOST missing' >&2 + exit $UNKNOWN +fi + +ssh sshmon@"$SSHMON_HOST" "sudo $SSHMON_COMMAND" + +exitcode=$? + +if [ "$exitcode" = 124 ] +then + echo 'check_by_sshmon: Timeout while running check remotely' >&2 + exit $UNKNOWN +elif [ "$exitcode" = 255 ] +then + echo 'check_by_sshmon: SSH error' >&2 + exit $UNKNOWN +else + exit $exitcode +fi diff --git a/bundles/icinga2/files/conf.d/api-users.conf b/bundles/icinga2/files/conf.d/api-users.conf new file mode 100644 index 0000000..e5d6972 --- /dev/null +++ b/bundles/icinga2/files/conf.d/api-users.conf @@ -0,0 +1,10 @@ +% for name, conf in sorted(users.items()): +object ApiUser "${name}" { + password = "${conf['password']}" + permissions = [ +% for permission in conf['permissions']: + "${permission}", +% endfor + ] +} +% endfor diff --git a/bundles/icinga2/files/conf.d/app.conf b/bundles/icinga2/files/conf.d/app.conf new file mode 100644 index 0000000..3e4be0d --- /dev/null +++ b/bundles/icinga2/files/conf.d/app.conf @@ -0,0 +1 @@ +object IcingaApplication "app" { } diff --git a/bundles/icinga2/files/conf.d/commands.conf b/bundles/icinga2/files/conf.d/commands.conf new file mode 100644 index 0000000..571ff35 --- /dev/null +++ b/bundles/icinga2/files/conf.d/commands.conf @@ -0,0 +1,198 @@ +/* Command objects */ + +/* Notification Commands + * + * Please check the documentation for all required and + * optional parameters. + */ + + +object CheckCommand "sshmon" { + import "ipv4-or-ipv6" + + command = [ "/usr/lib/nagios/plugins/check_by_sshmon" ] + + env.SSHMON_TEST = "1234" + env.SSHMON_COMMAND = "$command$" + env.SSHMON_HOST = "$address$" +} + + +object NotificationCommand "mail-host-notification" { + command = [ ConfigDir + "/scripts/mail-host-notification.sh" ] + + arguments += { + "-4" = "$notification_address$" + "-6" = "$notification_address6$" + "-b" = "$notification_author$" + "-c" = "$notification_comment$" + "-d" = { + required = true + value = "$notification_date$" + } + "-f" = { + value = "$notification_from$" + description = "Set from address. Requires GNU mailutils (Debian/Ubuntu) or mailx (RHEL/SUSE)" + } + "-i" = "$notification_icingaweb2url$" + "-l" = { + required = true + value = "$notification_hostname$" + } + "-n" = { + required = true + value = "$notification_hostdisplayname$" + } + "-o" = { + required = true + value = "$notification_hostoutput$" + } + "-r" = { + required = true + value = "$notification_useremail$" + } + "-s" = { + required = true + value = "$notification_hoststate$" + } + "-t" = { + required = true + value = "$notification_type$" + } + "-v" = "$notification_logtosyslog$" + } + + vars += { + notification_address = "$address$" + notification_address6 = "$address6$" + notification_author = "$notification.author$" + notification_comment = "$notification.comment$" + notification_type = "$notification.type$" + notification_date = "$icinga.long_date_time$" + notification_hostname = "$host.name$" + notification_hostdisplayname = "$host.display_name$" + notification_hostoutput = "$host.output$" + notification_hoststate = "$host.state$" + notification_useremail = "$user.email$" + } +} + +object NotificationCommand "mail-service-notification" { + command = [ ConfigDir + "/scripts/mail-service-notification.sh" ] + + arguments += { + "-4" = "$notification_address$" + "-6" = "$notification_address6$" + "-b" = "$notification_author$" + "-c" = "$notification_comment$" + "-d" = { + required = true + value = "$notification_date$" + } + "-e" = { + required = true + value = "$notification_servicename$" + } + "-f" = { + value = "$notification_from$" + description = "Set from address. Requires GNU mailutils (Debian/Ubuntu) or mailx (RHEL/SUSE)" + } + "-i" = "$notification_icingaweb2url$" + "-l" = { + required = true + value = "$notification_hostname$" + } + "-n" = { + required = true + value = "$notification_hostdisplayname$" + } + "-o" = { + required = true + value = "$notification_serviceoutput$" + } + "-r" = { + required = true + value = "$notification_useremail$" + } + "-s" = { + required = true + value = "$notification_servicestate$" + } + "-t" = { + required = true + value = "$notification_type$" + } + "-u" = { + required = true + value = "$notification_servicedisplayname$" + } + "-v" = "$notification_logtosyslog$" + } + + vars += { + notification_address = "$address$" + notification_address6 = "$address6$" + notification_author = "$notification.author$" + notification_comment = "$notification.comment$" + notification_type = "$notification.type$" + notification_date = "$icinga.long_date_time$" + notification_hostname = "$host.name$" + notification_hostdisplayname = "$host.display_name$" + notification_servicename = "$service.name$" + notification_serviceoutput = "$service.output$" + notification_servicestate = "$service.state$" + notification_useremail = "$user.email$" + notification_servicedisplayname = "$service.display_name$" + } +} + +/* + * If you prefer to use the notification scripts with environment + * variables instead of command line parameters, you can use + * the following commands. They have been updated from < 2.7 + * to support the new notification scripts and should help + * with an upgrade. + * Remove the comment blocks and comment the notification commands above. + */ + +/* + +object NotificationCommand "mail-host-notification" { + command = [ ConfigDir + "/scripts/mail-host-notification.sh" ] + + env = { + NOTIFICATIONTYPE = "$notification.type$" + HOSTDISPLAYNAME = "$host.display_name$" + HOSTNAME = "$host.name$" + HOSTADDRESS = "$address$" + HOSTSTATE = "$host.state$" + LONGDATETIME = "$icinga.long_date_time$" + HOSTOUTPUT = "$host.output$" + NOTIFICATIONAUTHORNAME = "$notification.author$" + NOTIFICATIONCOMMENT = "$notification.comment$" + HOSTDISPLAYNAME = "$host.display_name$" + USEREMAIL = "$user.email$" + } +} + +object NotificationCommand "mail-service-notification" { + command = [ ConfigDir + "/scripts/mail-service-notification.sh" ] + + env = { + NOTIFICATIONTYPE = "$notification.type$" + SERVICENAME = "$service.name$" + HOSTNAME = "$host.name$" + HOSTDISPLAYNAME = "$host.display_name$" + HOSTADDRESS = "$address$" + SERVICESTATE = "$service.state$" + LONGDATETIME = "$icinga.long_date_time$" + SERVICEOUTPUT = "$service.output$" + NOTIFICATIONAUTHORNAME = "$notification.author$" + NOTIFICATIONCOMMENT = "$notification.comment$" + HOSTDISPLAYNAME = "$host.display_name$" + SERVICEDISPLAYNAME = "$service.display_name$" + USEREMAIL = "$user.email$" + } +} + +*/ diff --git a/bundles/icinga2/files/conf.d/groups.conf b/bundles/icinga2/files/conf.d/groups.conf new file mode 100644 index 0000000..e6004a3 --- /dev/null +++ b/bundles/icinga2/files/conf.d/groups.conf @@ -0,0 +1,37 @@ +/** + * Host group examples. + */ + +object HostGroup "linux-servers" { + display_name = "Linux Servers" + + assign where host.vars.os == "Linux" +} + +object HostGroup "windows-servers" { + display_name = "Windows Servers" + + assign where host.vars.os == "Windows" +} + +/** + * Service group examples. + */ + +object ServiceGroup "ping" { + display_name = "Ping Checks" + + assign where match("ping*", service.name) +} + +object ServiceGroup "http" { + display_name = "HTTP Checks" + + assign where match("http*", service.check_command) +} + +object ServiceGroup "disk" { + display_name = "Disk Checks" + + assign where match("disk*", service.check_command) +} diff --git a/bundles/icinga2/files/conf.d/notifications.conf b/bundles/icinga2/files/conf.d/notifications.conf new file mode 100644 index 0000000..ac65875 --- /dev/null +++ b/bundles/icinga2/files/conf.d/notifications.conf @@ -0,0 +1,33 @@ +/** + * The example notification apply rules. + * + * Only applied if host/service objects have + * the custom variable `notification` defined + * and containing `mail` as key. + * + * Check `hosts.conf` for an example. + */ + +apply Notification "mail-icingaadmin" to Host { + import "mail-host-notification" + user_groups = host.vars.notification.mail.groups + users = host.vars.notification.mail.users + + //interval = 2h + + //vars.notification_logtosyslog = true + + assign where host.vars.notification.mail +} + +apply Notification "mail-icingaadmin" to Service { + import "mail-service-notification" + user_groups = host.vars.notification.mail.groups + users = host.vars.notification.mail.users + + //interval = 2h + + //vars.notification_logtosyslog = true + + assign where host.vars.notification.mail +} diff --git a/bundles/icinga2/files/conf.d/templates.conf b/bundles/icinga2/files/conf.d/templates.conf new file mode 100644 index 0000000..d3e1147 --- /dev/null +++ b/bundles/icinga2/files/conf.d/templates.conf @@ -0,0 +1,15 @@ +template Host "generic-host" { + max_check_attempts = 3 + check_interval = 1m + retry_interval = 30s + + check_command = "hostalive" +} + +template Service "generic-service" { + max_check_attempts = 5 + check_interval = 1m + retry_interval = 30s +} + +template User "generic-user" {} diff --git a/bundles/icinga2/files/conf.d/timeperiods.conf b/bundles/icinga2/files/conf.d/timeperiods.conf new file mode 100644 index 0000000..2d403bb --- /dev/null +++ b/bundles/icinga2/files/conf.d/timeperiods.conf @@ -0,0 +1,34 @@ +/** + * Sample timeperiods for Icinga 2. + * Check the documentation for details. + */ + +object TimePeriod "24x7" { + display_name = "Icinga 2 24x7 TimePeriod" + ranges = { + "monday" = "00:00-24:00" + "tuesday" = "00:00-24:00" + "wednesday" = "00:00-24:00" + "thursday" = "00:00-24:00" + "friday" = "00:00-24:00" + "saturday" = "00:00-24:00" + "sunday" = "00:00-24:00" + } +} + +object TimePeriod "9to5" { + display_name = "Icinga 2 9to5 TimePeriod" + ranges = { + "monday" = "09:00-17:00" + "tuesday" = "09:00-17:00" + "wednesday" = "09:00-17:00" + "thursday" = "09:00-17:00" + "friday" = "09:00-17:00" + } +} + +object TimePeriod "never" { + display_name = "Icinga 2 never TimePeriod" + ranges = { + } +} diff --git a/bundles/icinga2/files/constants.conf b/bundles/icinga2/files/constants.conf new file mode 100644 index 0000000..007c0d1 --- /dev/null +++ b/bundles/icinga2/files/constants.conf @@ -0,0 +1,6 @@ +const PluginDir = "/usr/lib/nagios/plugins" +const ManubulonPluginDir = "/usr/lib/nagios/plugins" +const PluginContribDir = "/usr/lib/nagios/plugins" +const NodeName = "${hostname}" +const ZoneName = NodeName +const TicketSalt = "" diff --git a/bundles/icinga2/files/features/api.conf b/bundles/icinga2/files/features/api.conf new file mode 100644 index 0000000..869188e --- /dev/null +++ b/bundles/icinga2/files/features/api.conf @@ -0,0 +1 @@ +object ApiListener "api" {} diff --git a/bundles/icinga2/files/features/checker.conf b/bundles/icinga2/files/features/checker.conf new file mode 100644 index 0000000..437ff0c --- /dev/null +++ b/bundles/icinga2/files/features/checker.conf @@ -0,0 +1 @@ +object CheckerComponent "checker" { } diff --git a/bundles/icinga2/files/features/ido-pgsql.conf b/bundles/icinga2/files/features/ido-pgsql.conf new file mode 100644 index 0000000..bbdf288 --- /dev/null +++ b/bundles/icinga2/files/features/ido-pgsql.conf @@ -0,0 +1,8 @@ +library "db_ido_pgsql" + +object IdoPgsqlConnection "ido-pgsql" { + user = "icinga2", + password = "${db_password}", + host = "localhost", + database = "icinga2" +} diff --git a/bundles/icinga2/files/features/notification.conf b/bundles/icinga2/files/features/notification.conf new file mode 100644 index 0000000..0a8fb18 --- /dev/null +++ b/bundles/icinga2/files/features/notification.conf @@ -0,0 +1 @@ +object NotificationComponent "notification" { } diff --git a/bundles/icinga2/files/features/syslog.conf b/bundles/icinga2/files/features/syslog.conf new file mode 100644 index 0000000..161fd36 --- /dev/null +++ b/bundles/icinga2/files/features/syslog.conf @@ -0,0 +1,3 @@ +object SyslogLogger "syslog" { + severity = "warning" +} diff --git a/bundles/icinga2/files/hosts.d/host.conf b/bundles/icinga2/files/hosts.d/host.conf new file mode 100644 index 0000000..2f04906 --- /dev/null +++ b/bundles/icinga2/files/hosts.d/host.conf @@ -0,0 +1,38 @@ +<%! + from bundlewrap.utils import Fault + + def render_value(key, value): + if isinstance(value, Fault): + return render_value(key, value.value) + elif isinstance(value, type(None)): + return '""' + elif isinstance(value, bool): + return 'true' if value else 'false' + elif isinstance(value, int): + return str(value) + elif isinstance(value, str): + if key.endswith('_interval'): + return value + else: + return f'"{value}"' + elif isinstance(value, (list, set)): + return '[' + ', '.join(render_value(e) for e in sorted(value)) + ']' + else: + raise Exception(f"cant process type '{type(value)}' of value '{value}'") +%> + +object Host "${host_name}" { + import "generic-host" + % for key, value in sorted(host_settings.items()): + ${key} = ${render_value(key, value)} + % endfor +} + +% for service_name, service_config in sorted(services.items()): +object Service "${service_name}" { + import "generic-service" + % for key, value in sorted(service_config.items()): + ${key} = ${render_value(key, value)} + % endfor +} +% endfor diff --git a/bundles/icinga2/files/icinga2.conf b/bundles/icinga2/files/icinga2.conf new file mode 100644 index 0000000..94f2734 --- /dev/null +++ b/bundles/icinga2/files/icinga2.conf @@ -0,0 +1,10 @@ +include "constants.conf" +include "zones.conf" + +include +include +include + +include "features.d/*.conf" +include_recursive "conf.d" +include "hosts.d/*.conf" diff --git a/bundles/icinga2/files/zones.conf b/bundles/icinga2/files/zones.conf new file mode 100644 index 0000000..59b1fdb --- /dev/null +++ b/bundles/icinga2/files/zones.conf @@ -0,0 +1,7 @@ +object Endpoint NodeName { + host = NodeName +} + +object Zone ZoneName { + endpoints = [ NodeName ] +} diff --git a/bundles/icinga2/items.py b/bundles/icinga2/items.py new file mode 100644 index 0000000..72f48b4 --- /dev/null +++ b/bundles/icinga2/items.py @@ -0,0 +1,259 @@ +from ipaddress import ip_interface + +directories = { + '/etc/icinga2': { + 'purge': True, + 'owner': 'nagios', + 'group': 'nagios', + 'mode': '0750', + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, + '/etc/icinga2/conf.d': { + 'purge': True, + 'owner': 'nagios', + 'group': 'nagios', + 'mode': '0750', + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, + '/etc/icinga2/hosts.d': { + 'purge': True, + 'owner': 'nagios', + 'group': 'nagios', + 'mode': '0750', + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, + '/etc/icinga2/features.d': { + 'purge': True, + 'owner': 'nagios', + 'group': 'nagios', + 'mode': '0750', + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, + '/etc/icinga2/scripts': { + 'owner': 'nagios', + 'group': 'nagios', + 'mode': '0750', + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, + '/var/lib/icinga2': { + 'owner': 'nagios', + 'group': 'nagios', + 'mode': '0750', + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, + '/var/lib/icinga2/certs': { + 'owner': 'nagios', + 'group': 'nagios', + 'mode': '0700', + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, +} + +files = { + '/etc/icinga2/icinga2.conf': { + 'owner': 'nagios', + 'group': 'nagios', + 'mode': '0640', + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, + '/etc/icinga2/constants.conf': { + 'content_type': 'mako', + 'owner': 'nagios', + 'group': 'nagios', + 'mode': '0640', + 'context': { + 'hostname': node.metadata.get('icinga2/hostname') + }, + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, + '/etc/icinga2/zones.conf': { + 'content_type': 'mako', + 'context': { + 'hostname': node.metadata.get('icinga2/hostname') + }, + 'owner': 'nagios', + 'group': 'nagios', + 'mode': '0640', + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, + '/etc/icinga2/conf.d/api-users.conf': { + 'source': 'conf.d/api-users.conf', + 'content_type': 'mako', + 'owner': 'nagios', + 'group': 'nagios', + 'mode': '0640', + 'context': { + 'users': node.metadata.get('icinga2/api_users'), + }, + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, + '/etc/icinga2/conf.d/app.conf': { + 'source': 'conf.d/app.conf', + 'content_type': 'mako', + 'owner': 'nagios', + 'group': 'nagios', + 'mode': '0640', + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, + '/etc/icinga2/conf.d/commands.conf': { + 'source': 'conf.d/commands.conf', + 'content_type': 'mako', + 'owner': 'nagios', + 'group': 'nagios', + 'mode': '0640', + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, + '/etc/icinga2/conf.d/groups.conf': { + 'source': 'conf.d/groups.conf', + 'content_type': 'mako', + 'owner': 'nagios', + 'group': 'nagios', + 'mode': '0640', + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, + '/etc/icinga2/conf.d/notifications.conf': { + 'source': 'conf.d/notifications.conf', + 'content_type': 'mako', + 'owner': 'nagios', + 'group': 'nagios', + 'mode': '0640', + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, + '/etc/icinga2/conf.d/templates.conf': { + 'source': 'conf.d/templates.conf', + 'content_type': 'mako', + 'owner': 'nagios', + 'group': 'nagios', + 'mode': '0640', + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, + '/etc/icinga2/conf.d/timeperiods.conf': { + 'source': 'conf.d/timeperiods.conf', + 'content_type': 'mako', + 'owner': 'nagios', + 'group': 'nagios', + 'mode': '0640', + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, + '/etc/icinga2/features.d/ido-pgsql.conf': { + 'source': 'features/ido-pgsql.conf', + 'content_type': 'mako', + 'owner': 'nagios', + 'group': 'nagios', + 'context': { + 'db_password': node.metadata.get('postgresql/roles/icinga2/password') + }, + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, + '/etc/icinga2/features.d/syslog.conf': { + 'source': 'features/syslog.conf', + 'owner': 'nagios', + 'group': 'nagios', + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, + '/etc/icinga2/features.d/notification.conf': { + 'source': 'features/notification.conf', + 'owner': 'nagios', + 'group': 'nagios', + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, + '/etc/icinga2/features.d/checker.conf': { + 'source': 'features/checker.conf', + 'owner': 'nagios', + 'group': 'nagios', + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, + '/etc/icinga2/features.d/api.conf': { + 'source': 'features/api.conf', + 'owner': 'nagios', + 'group': 'nagios', + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, + '/var/lib/icinga2/certs/ca.crt': { + 'content_type': 'download', + 'source': f'https://letsencrypt.org/certs/isrg-root-x1-cross-signed.pem', + 'owner': 'nagios', + 'group': 'nagios', + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + }, + '/usr/lib/nagios/plugins/check_by_sshmon': { + 'mode': '0755', + }, +} + +for other_node in repo.nodes: + if other_node.dummy: + continue + elif not other_node.in_group('monitored'): + continue + + files[f'/etc/icinga2/hosts.d/{other_node.name}.conf'] = { + 'content_type': 'mako', + 'source': 'hosts.d/host.conf', + 'owner': 'nagios', + 'context': { + 'host_name': other_node.name, + 'host_settings': { + 'address': str(ip_interface(other_node.metadata.get('network/internal/ipv4', None) or other_node.metadata.get('wireguard/my_ip')).ip), + }, + 'services': other_node.metadata.get('monitoring/services'), + }, + 'triggers': [ + 'svc_systemd:icinga2.service:restart', + ], + } + +svc_systemd = { + 'icinga2.service': { + 'needs': [ + 'pkg_apt:icinga2-ido-pgsql', + 'svc_systemd:postgresql', + ], + }, +} diff --git a/bundles/icinga2/metadata.py b/bundles/icinga2/metadata.py new file mode 100644 index 0000000..8295ca6 --- /dev/null +++ b/bundles/icinga2/metadata.py @@ -0,0 +1,73 @@ +from hashlib import sha3_256 + +defaults = { + 'apt': { + 'packages': { + 'icinga2': {}, + 'icinga2-ido-pgsql': {}, + 'icingacli': {}, + 'monitoring-plugins': {}, + }, + 'sources': { + 'deb https://packages.icinga.com/debian icinga-{release} main', + }, + }, + 'icinga2': { + 'api_users': { + 'root': { + 'password': repo.vault.password_for(f'icinga2 api user root on {node.name}'), + 'permissions': {'*'}, + } + }, + }, + 'postgresql': { + 'databases': { + 'icinga2': { + 'owner': 'icinga2', + }, + }, + 'roles': { + 'icinga2': { + 'password': repo.vault.password_for(f'psql icinga2 on {node.name}'), + }, + }, + }, + 'users': { + 'nagios': { + 'home': '/var/lib/nagios', + 'shell': '/usr/sbin/nologin', + }, + }, + 'zfs': { + 'datasets': { + 'tank/icinga2': { + 'mountpoint': '/var/lib/icinga2', + 'needed_by': { + 'pkg_apt:icinga2', + 'pkg_apt:icingaweb2', + 'pkg_apt:icinga2-ido-pgsql', + }, + }, + }, + }, +} + + +@metadata_reactor.provides( + 'nginx/vhosts', +) +def letsencrypt(metadata): + return { + 'letsencrypt': { + 'domains': { + metadata.get('icingaweb2/hostname'): { + 'reload': {'icinga2'}, + 'owner': 'nagios', + 'group': 'nagios', + 'location': '/var/lib/icinga2/certs', + 'privkey_name': metadata.get('hostname') + '.key', + 'cert_name': metadata.get('hostname') + '.crt', + }, + }, + }, + } diff --git a/bundles/icingaweb2/README.md b/bundles/icingaweb2/README.md new file mode 100644 index 0000000..18e3c30 --- /dev/null +++ b/bundles/icingaweb2/README.md @@ -0,0 +1,4 @@ +- apply +- open /icingaweb2/setup in browser +- fill in values from metadata +- apply diff --git a/bundles/icingaweb2/items.py b/bundles/icingaweb2/items.py new file mode 100644 index 0000000..44a188c --- /dev/null +++ b/bundles/icingaweb2/items.py @@ -0,0 +1,63 @@ +directories = { + '/etc/icingaweb2': { +# 'purge': True, + 'owner': 'www-data', + 'group': 'icingaweb2', + 'mode': '2770', + }, + '/etc/icingaweb2/enabledModules': { +# 'purge': True, + 'owner': 'www-data', + 'group': 'icingaweb2', + 'mode': '2770', + }, + '/etc/icingaweb2/modules': { +# 'purge': True, + 'owner': 'www-data', + 'group': 'icingaweb2', + 'mode': '2770', + }, +} + +files = { + '/etc/icingaweb2/setup.token': { + 'content': node.metadata.get('icingaweb2/setup_token'), + 'owner': 'www-data', + 'group': 'icingaweb2', + 'mode': '0660', + }, +} + +symlinks = { + '/etc/icingaweb2/enabledModules/monitoring': { + 'target': '/usr/share/icingaweb2/modules/monitoring', + 'owner': 'www-data', + 'group': 'icingaweb2', + }, +} + +for name in [ + 'authentication.ini', + 'config.ini', + 'groups.ini', + 'resources.ini', + 'roles.ini', +]: + files[f'/etc/icingaweb2/{name}'] = { + 'content': repo.libs.ini.dumps(node.metadata.get(f'icingaweb2/{name}')), + 'owner': 'www-data', + 'group': 'icingaweb2', + 'mode': '0660', + } + +for name in [ + 'config.ini', + 'backends.ini', + 'commandtransports.ini', +]: + files[f'/etc/icingaweb2/modules/monitoring/{name}'] = { + 'content': repo.libs.ini.dumps(node.metadata.get(f'icingaweb2/monitoring/{name}')), + 'owner': 'www-data', + 'group': 'icingaweb2', + 'mode': '0660', + } diff --git a/bundles/icingaweb2/metadata.py b/bundles/icingaweb2/metadata.py new file mode 100644 index 0000000..a0b1909 --- /dev/null +++ b/bundles/icingaweb2/metadata.py @@ -0,0 +1,171 @@ +from hashlib import sha3_256 + +defaults = { + 'apt': { + 'packages': { + 'icingaweb2': {}, + 'php-ldap': {}, + 'php-json': {}, + 'php-intl': {}, + 'php-xml': {}, + 'php-gd': {}, + 'php-imagick': {}, + 'php-pgsql': {}, + 'icingaweb2-module-monitoring': {}, + }, + 'sources': { + 'deb https://packages.icinga.com/debian icinga-{release} main', + 'deb https://packages.icinga.com/debian icinga-{release}-snapshots main', + }, + }, + 'icingaweb2': { + 'authentication.ini': { + 'icingaweb2': { + 'backend': 'db', + 'resource': 'icingaweb2_db', + }, + }, + 'config.ini': { + 'global': { + 'show_stacktraces': '1', + 'show_application_state_messages': '1', + 'module_path': '/usr/share/icingaweb2/modules', + 'config_backend': 'db', + 'config_resource': 'icingaweb2_db', + }, + 'logging': { + 'log': 'syslog', + 'level': 'INFO', + 'application': 'icingaweb2', + 'facility': 'user', + }, + }, + 'groups.ini': { + 'icingaweb2': { + 'backend': 'db', + 'resource': 'icingaweb2_db', + }, + }, + 'resources.ini': { + 'icingaweb2_db': { + 'type': 'db', + 'db': 'pgsql', + 'host': 'localhost', + 'port': '5432', + 'dbname': 'icingaweb2', + 'username': 'icingaweb2', + 'password': str(repo.vault.password_for(f'psql icingaweb2 on {node.name}')), + 'charset': '', + 'use_ssl': '0', + }, + 'icinga_ido': { + 'type': 'db', + 'db': 'pgsql', + 'host': 'localhost', + 'port': '5432', + 'dbname': 'icinga2', + 'username': 'icinga2', + 'charset': '', + 'use_ssl': '0', + }, + }, + 'roles.ini': { + 'Administrators': { + 'users': 'root', + 'permissions': '*', + 'groups': 'Administrators', + }, + }, + 'monitoring': { + 'config.ini': { + 'security': { + 'protected_customvars': '*pw*,*pass*,community', + }, + }, + 'backends.ini': { + 'icinga2': { + 'type': 'ido', + 'resource': 'icinga_ido', + }, + }, + 'commandtransports.ini': { + 'icinga2': { + 'transport': 'api', + 'host': 'localhost', + 'port': '5665', + 'username': 'root', + }, + }, + }, + }, + 'postgresql': { + 'databases': { + 'icingaweb2': { + 'owner': 'icingaweb2', + }, + }, + 'roles': { + 'icingaweb2': { + 'password': str(repo.vault.password_for(f'psql icingaweb2 on {node.name}')), + }, + }, + }, + 'redis': { + 'icingaweb2': {}, + }, +} + + +@metadata_reactor.provides( + 'icingaweb2/hostname', + 'icingaweb2/resources.ini/icinga_ido/icinga2/password', + 'icingaweb2/monitoring/commandtransports.ini/icinga2/password', +) +def stuff(metadata): + return { + 'icingaweb2': { + 'hostname': metadata.get('icinga2/hostname'), + 'resources.ini': { + 'icinga_ido': { + 'password': str(metadata.get('postgresql/roles/icinga2/password')), + }, + }, + 'monitoring': { + 'commandtransports.ini': { + 'icinga2': { + 'password': str(metadata.get('icinga2/api_users/root/password')), + }, + }, + }, + }, + } + + +@metadata_reactor.provides( + 'icingaweb2/setup_token', + 'icingaweb2/root_password', +) +def setup_token(metadata): + return { + 'icingaweb2': { + 'setup_token': sha3_256(metadata.get('id').encode()).hexdigest()[:16], + 'root_password': str(repo.vault.password_for(f"icingaweb2 root user on {metadata.get('id')}")), + }, + } + + +@metadata_reactor.provides( + 'nginx/vhosts', +) +def nginx(metadata): + return { + 'nginx': { + 'vhosts': { + metadata.get('icingaweb2/hostname'): { + 'content': 'icingaweb2/vhost.conf', + 'context': { + }, + }, + }, + }, + } diff --git a/bundles/letsencrypt/files/hook.sh b/bundles/letsencrypt/files/hook.sh index ca33f8a..f1d00e7 100644 --- a/bundles/letsencrypt/files/hook.sh +++ b/bundles/letsencrypt/files/hook.sh @@ -26,19 +26,23 @@ deploy_cert() { CERTFILE="$3" FULLCHAINFILE="$4" CHAINFILE="$5" - + case $DOMAIN in % for domain, conf in sorted(domains.items()): <% if not conf: continue %>\ ${domain}) % if conf.get('location', None): - cat "$KEYFILE" > "${conf['location']}/privkey.pem" - cat "$CERTFILE" > "${conf['location']}/cert.pem" - cat "$FULLCHAINFILE" > "${conf['location']}/fullchain.pem" - cat "$CHAINFILE" > "${conf['location']}/chain.pem" + cat "$KEYFILE" > "${conf['location']}/${conf.get('privkey_name', 'privkey.pem')}" + cat "$CERTFILE" > "${conf['location']}/${conf.get('cert_name', 'cert.pem')}" + cat "$FULLCHAINFILE" > "${conf['location']}/${conf.get('fullchain_name', 'fullchain.pem')}" + cat "$CHAINFILE" > "${conf['location']}/${conf.get('chain_name', 'chain.pem')}" % endif % if conf.get('owner', None): - chown ${conf['owner']} "${conf['location']}/privkey.pem" "${conf['location']}/cert.pem" "${conf['location']}/fullchain.pem" "${conf['location']}/chain.pem" + chown ${conf['owner']}:${conf.get('group', '')} \ + "${conf['location']}/${conf.get('privkey_name', 'privkey.pem')}" \ + "${conf['location']}/${conf.get('cert_name', 'cert.pem')}" \ + "${conf['location']}/${conf.get('fullchain_name', 'fullchain.pem')}" \ + "${conf['location']}/${conf.get('chain_name', 'chain.pem')}" % endif % for service in sorted(conf.get('reload', [])): systemctl reload-or-restart ${service} diff --git a/bundles/monitored/metadata.py b/bundles/monitored/metadata.py new file mode 100644 index 0000000..f6d4cb7 --- /dev/null +++ b/bundles/monitored/metadata.py @@ -0,0 +1,55 @@ +defaults = { + 'monitoring': { + 'services': { + 'test': { + 'vars.command': '/bin/ls /', + }, + }, + }, +} + + +@metadata_reactor.provides( + 'monitoring/services', +) +def default_check_command(metadata): + services = {} + + for name, conf in metadata.get('monitoring/services').items(): + services[name] = {} + + if 'host_name' not in conf: + services[name]['host_name'] = node.name + + if 'check_command' not in conf: + services[name]['check_command'] = 'sshmon' + + return { + 'monitoring': { + 'services': services, + }, + } + + + +@metadata_reactor.provides( + 'users/sshmon/authorized_users', + 'sudoers/sshmon', +) +def user(metadata): + return { + 'users': { + 'sshmon': { + 'authorized_users': { + 'nagios@' + metadata.get('monitoring/icinga2_node'), + } + }, + }, + 'sudoers': { + 'sshmon': { + conf['vars.command'] + for conf in metadata.get('monitoring/services').values() + if conf['check_command'] == 'sshmon' + }, + }, + } diff --git a/bundles/systemd-timers/files/check_systemd_timer b/bundles/systemd-timers/files/check_systemd_timer new file mode 100644 index 0000000..d3f2227 --- /dev/null +++ b/bundles/systemd-timers/files/check_systemd_timer @@ -0,0 +1,14 @@ +Timer=$1 + +Triggers=$(systemctl show ${Timer}.timer --property=Triggers --value) +if systemctl is-failed "$Triggers" +then + InvocationID=$(systemctl show "$Triggers" --property=InvocationID --value) + ExitCode=$(systemctl show "$Triggers" -p ExecStartEx --value | sed 's/^{//' | sed 's/}$//' | tr ';' '\n' | xargs -n 1 | grep '^status=' | cut -d '=' -f 2) + journalctl INVOCATION_ID="$InvocationID" --output cat # timer invocation output + echo "-----------------" + journalctl _SYSTEMD_INVOCATION_ID="$InvocationID" --output cat -n 32 # service invocation output + exit 1 +else + exit 0 +fi diff --git a/bundles/systemd-timers/items.py b/bundles/systemd-timers/items.py index 697c1fa..f53c496 100644 --- a/bundles/systemd-timers/items.py +++ b/bundles/systemd-timers/items.py @@ -2,3 +2,7 @@ svc_systemd['cron'] = { 'enabled': False, 'running': False, } + +files['/usr/lib/nagios/plugins/check_systemd_timer'] = { + 'mode': '0755', +} diff --git a/bundles/systemd-timers/metadata.py b/bundles/systemd-timers/metadata.py index 230c950..50d298c 100644 --- a/bundles/systemd-timers/metadata.py +++ b/bundles/systemd-timers/metadata.py @@ -48,3 +48,19 @@ def systemd(metadata): 'services': services, }, } + + +@metadata_reactor.provides( + 'monitoring/services', +) +def monitoring(metadata): + return { + 'monitoring': { + 'services': { + f'{name}.timer': { + 'vars.command': f'/usr/lib/nagios/plugins/check_systemd_timer {name}' + } + for name in metadata.get('systemd-timers') + }, + }, + } diff --git a/data/apt/keys/packages.icinga.com.asc b/data/apt/keys/packages.icinga.com.asc new file mode 100644 index 0000000..901c78c --- /dev/null +++ b/data/apt/keys/packages.icinga.com.asc @@ -0,0 +1,30 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v2.0.19 (GNU/Linux) + +mQGiBFKHzk4RBACSHMIFTtfw4ZsNKAA03Gf5t7ovsKWnS7kcMYleAidypqhOmkGg +0petiYsMPYT+MOepCJFGNzwQwJhZrdLUxxMSWay4Xj0ArgpD9vbvU+gj8Tb02l+x +SqNGP8jXMV5UnK4gZsrYGLUPvx47uNNYRIRJAGOPYTvohhnFJiG402dzlwCg4u5I +1RdFplkp9JM6vNM9VBIAmcED/2jr7UQGsPs8YOiPkskGHLh/zXgO8SvcNAxCLgbp +BjGcF4Iso/A2TAI/2KGJW6kBW/Paf722ltU6s/6mutdXJppgNAz5nfpEt4uZKZyu +oSWf77179B2B/Wl1BsX/Oc3chscAgQb2pD/qPF/VYRJU+hvdQkq1zfi6cVsxyREV +k+IwA/46nXh51CQxE29ayuy1BoIOxezvuXFUXZ8rP6aCh4KaiN9AJoy7pBieCzsq +d7rPEeGIzBjI+yhEu8p92W6KWzL0xduWfYg9I7a2GTk8CaLX2OCLuwnKd7RVDyyZ +yzRjWs0T5U7SRAWspLStYxMdKert9lLyQiRHtLwmlgBPqa0gh7Q+SWNpbmdhIE9w +ZW4gU291cmNlIE1vbml0b3JpbmcgKEJ1aWxkIHNlcnZlcikgPGluZm9AaWNpbmdh +Lm9yZz6IYAQTEQIAIAUCUofOTgIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJ +EMbjGcM0QQaCgSQAnRjXdbsyqziqhmxfAKffNJYuMPwdAKCS/IRCVyQzApFBtIBQ +1xuoym/4C7kCDQRSh85OEAgAvPwjlURCi8z6+7i60no4n16dNcSzd6AT8Kizpv2r +9BmNBff/GNYGnHyob/DMtmO2esEuVG8w62rO9m1wzzXzjbtmtU7NZ1Tg+C+reU2I +GNVu3SYtEVK/UTJHAhLcgry9yD99610tYPN2Fx33Efse94mXOreBfCvDsmFGSc7j +GVNCWXpMR3jTYyGj1igYd5ztOzG63D8gPyOucTTl+RWN/G9EoGBv6sWqk5eCd1Fs +JlWyQX4BJn3YsCZx3uj1DWL0dAl2zqcn6m1M4oj1ozW47MqM/efKOcV6VvCs9SL8 +F/NFvZcH4LKzeupCQ5jEONqcTlVlnLlIqId95Z4DI4AV9wADBQf/S6sKA4oH49tD +Yb5xAfUyEp5ben05TzUJbXs0Z7hfRQzy9+vQbWGamWLgg3QRUVPx1e4IT+W5vEm5 +dggNTMEwlLMI7izCPDcD32B5oxNVxlfj428KGllYWCFj+edY+xKTvw/PHnn+drKs +LE65Gwx4BPHm9EqWHIBX6aPzbgbJZZ06f6jWVBi/N7e/5n8lkxXqS23DBKemapyu +S1i56sH7mQSMaRZP/iiOroAJemPNxv1IQkykxw2woWMmTLKLMCD/i+4DxejE50tK +dxaOLTc4HDCsattw/RVJO6fwE414IXHMv330z4HKWJevMQ+CmQGfswvCwgeBP9n8 +PItLjBQAXIhJBBgRAgAJBQJSh85OAhsMAAoJEMbjGcM0QQaCzpAAmwUNoRyySf9p +5G3/2UD1PMueIwOtAKDVVDXEq5LJPVg4iafNu0SRMwgP0Q== +=icbY +-----END PGP PUBLIC KEY BLOCK----- diff --git a/data/icingaweb2/vhost.conf b/data/icingaweb2/vhost.conf new file mode 100644 index 0000000..9d5030b --- /dev/null +++ b/data/icingaweb2/vhost.conf @@ -0,0 +1,31 @@ +# icingacli setup config webserver nginx --document-root /usr/share/icingaweb2/public --config /etc/icingaweb2 --fpm-uri 127.0.0.1:9000 + +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + + server_name ${server_name}; + root /usr/share/icingaweb2/public; + + ssl_certificate /var/lib/dehydrated/certs/${server_name}/fullchain.pem; + ssl_certificate_key /var/lib/dehydrated/certs/${server_name}/privkey.pem; + + location / { + return 302 /icingaweb2/index.php; + } + + location ~ ^/icingaweb2/index\.php(.*)$ { + fastcgi_pass unix:/run/php/php7.4-fpm.sock; + fastcgi_index index.php; + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME /usr/share/icingaweb2/public/index.php; + fastcgi_param ICINGAWEB_CONFIGDIR /etc/icingaweb2; + fastcgi_param REMOTE_USER $remote_user; + } + + location ~ ^/icingaweb2(.+)? { + alias /usr/share/icingaweb2/public; + index index.php; + try_files $1 $uri $uri/ /icingaweb2/index.php$is_args$args; + } +} diff --git a/data/nginx/proxy_pass.conf b/data/nginx/proxy_pass.conf index 16476ca..7d3069f 100644 --- a/data/nginx/proxy_pass.conf +++ b/data/nginx/proxy_pass.conf @@ -2,7 +2,7 @@ server { listen 443 ssl http2; listen [::]:443 ssl http2; server_name ${server_name}; - + ssl_certificate /var/lib/dehydrated/certs/${server_name}/fullchain.pem; ssl_certificate_key /var/lib/dehydrated/certs/${server_name}/privkey.pem; diff --git a/groups/applications/monitored.py b/groups/applications/monitored.py index ccf4ee8..bfbdfb5 100644 --- a/groups/applications/monitored.py +++ b/groups/applications/monitored.py @@ -1,10 +1,14 @@ { 'bundles': [ 'telegraf', + 'monitored', ], 'metadata': { 'telegraf': { 'influxdb_node': 'home.server', }, + 'monitoring': { + 'icinga2_node': 'home.server', + }, }, } diff --git a/nodes/home.server.py b/nodes/home.server.py index 5fbf9b0..ed706b9 100644 --- a/nodes/home.server.py +++ b/nodes/home.server.py @@ -18,7 +18,8 @@ 'gitea', # 'gollum', 'grafana', - 'grub', + 'icinga2', + 'icingaweb2', 'influxdb2', 'mirror', 'postgresql', @@ -73,6 +74,12 @@ 'grub': { 'kernel_params': {'nomodeset'}, # nvidia GT1030 freeze fix }, + 'icinga2': { + 'hostname': 'icinga.sublimity.de', + }, + 'icingaweb2': { + 'hostname': 'icinga.sublimity.de', + }, 'influxdb': { 'hostname': 'influxdb.sublimity.de', 'admin_token': '!decrypt:encrypt$gAAAAABg3z5PcaLYmUpcElJ07s_G-iYwnS8d532TcR8xUYbZfttT-B736zgR6J726mzKAFNYlIfJ7amNLIzi2ETDH5TAXWsOiAKpX8WC_dPBAvG3uXGtcPYENjdeuvllSagZzPt0hCIZQZXg--Z_YvzaX9VzNrVAgGD-sXQnghN5_Vhf9gVxxwP---VB_6iNlsf61Nc4axoS',