commit 572e29e723856e291169073b51d584b67e1d6d73 Author: mwiegand Date: Fri Jun 11 13:30:57 2021 +0200 wip diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a258834 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.secrets.cfg* diff --git a/bundles/gitea/files/app.ini b/bundles/gitea/files/app.ini new file mode 100644 index 0000000..e69de29 diff --git a/bundles/gitea/items.py b/bundles/gitea/items.py new file mode 100644 index 0000000..a690e49 --- /dev/null +++ b/bundles/gitea/items.py @@ -0,0 +1,55 @@ +downloads = { + '/usr/local/bin/gitea': { + 'url': 'https://dl.gitea.io/gitea/{version}/gitea-{version}-linux-amd64'.format(version=node.metadata['gitea']['version']), + 'sha256': node.metadata['gitea']['sha256'], + 'triggers': { + 'svc_systemd:gitea:restart', + }, + 'preceded_by': { + 'action:stop_gitea', + }, + }, +} + +users = { + 'git': {}, +} + +directories = { + '/home/git': { + 'mode': '0755', + 'owner': 'git', + 'group': 'git', + }, + '/var/lib/gitea': { + 'owner': 'git', + 'mode': '0700', + 'triggers': { + 'svc_systemd:gitea:restart', + }, + }, +} + +actions = { + 'chmod_gitea': { + 'command': 'chmod a+x /usr/local/bin/gitea', + 'unless': 'test -x /usr/local/bin/gitea', + 'needs': { + 'download:/usr/local/bin/gitea', + }, + }, + 'stop_gitea': { + 'command': 'systemctl stop gitea', + 'triggered': True, + }, +} + +files = { + '/etc/gitea/app.ini': { + 'content_type': 'mako', + 'context': node.metadata['gitea'], + 'triggers': { + 'svc_systemd:gitea:restart', + }, + }, +} diff --git a/bundles/gitea/metadata.py b/bundles/gitea/metadata.py new file mode 100644 index 0000000..db12542 --- /dev/null +++ b/bundles/gitea/metadata.py @@ -0,0 +1,102 @@ +defaults = { + 'gitea': { + 'database': { + 'username': 'gitea', + 'password': repo.vault.password_for('{} postgresql gitea'.format(node.name)), + 'database': 'gitea', + }, + 'app_name': 'Gitea', + 'lfs_secret_key': repo.vault.password_for('{} gitea lfs_secret_key'.format(node.name)), + 'security_secret_key': repo.vault.password_for('{} gitea security_secret_key'.format(node.name)), + 'oauth_secret_key': repo.vault.password_for('{} gitea oauth_secret_key'.format(node.name)), + 'internal_token': repo.vault.password_for('{} gitea internal_token'.format(node.name)), + }, + 'postgresql': { + 'roles': { + 'gitea': { + 'password': repo.vault.password_for('{} postgresql gitea'.format(node.name)), + }, + }, + 'databases': { + 'gitea': { + 'owner': 'gitea', + }, + }, + }, + 'systemd': { + 'services': { + 'gitea': { + 'content': { + 'Unit': { + 'Description': 'gitea', + 'After': 'syslog.target', + 'After': 'network.target', + 'Requires': 'postgresql.service', + }, + 'Service': { + 'RestartSec': '2s', + 'Type': 'simple', + 'User': 'git', + 'Group': 'git', + 'WorkingDirectory': '/var/lib/gitea/', + 'ExecStart': '/usr/local/bin/gitea web -c /etc/gitea/app.ini', + 'Restart': 'always', + 'Environment': 'USER=git HOME=/home/git GITEA_WORK_DIR=/var/lib/gitea', + }, + 'Install': { + 'WantedBy': 'multi-user.target', + }, + }, + 'needs': { + 'action:chmod_gitea', + 'download:/usr/local/bin/gitea', + 'file:/etc/systemd/system/gitea.service', + 'file:/etc/gitea/app.ini', + }, + }, + }, + }, +} + + +@metadata_reactor.provides( + 'nginx/vhosts', +) +def nginx(metadata): + if not node.has_bundle('nginx'): + raise DoNotRunAgain + + return { + 'nginx': { + 'vhosts': { + metadata.get('gitea/domain'): { + 'proxy': { + '/': { + 'target': 'http://127.0.0.1:22000', + }, + }, + 'website_check_path': '/user/login', + 'website_check_string': 'Sign In', + }, + }, + }, + } + + +@metadata_reactor.provides( + 'icinga2_api/gitea/services', +) +def icinga_check_for_new_release(metadata): + return { + 'icinga2_api': { + 'gitea': { + 'services': { + 'GITEA UPDATE': { + 'command_on_monitored_host': '/usr/local/share/icinga/plugins/check_github_for_new_release go-gitea/gitea v{}'.format(metadata.get('gitea/version')), + 'vars.notification.mail': True, + 'check_interval': '60m', + }, + }, + }, + }, + } diff --git a/bundles/l4d2/files/l4d2.service b/bundles/l4d2/files/l4d2.service new file mode 100644 index 0000000..06cc1fd --- /dev/null +++ b/bundles/l4d2/files/l4d2.service @@ -0,0 +1,13 @@ +GNU nano 4.8 /etc/systemd/system/l4d2-server-a.service +[Unit] +Description=l4d2 Server A +After=network.target steam-update.service + +[Service] +User=steam +WorkingDirectory=/home/steam/steam/l4d2 +ExecStart=/home/steam/steam/l4d2/srcds_run -port 27001 -secure +exec server_a.cfg +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/bundles/l4d2/files/update_addons b/bundles/l4d2/files/update_addons new file mode 100644 index 0000000..6025c5b --- /dev/null +++ b/bundles/l4d2/files/update_addons @@ -0,0 +1,4 @@ +#!/bin/bash + +/home/steam/steam_workshop_downloader/workshop.py -o /home/steam/steam/l4d2/left4dead2/addons 2283884609 +chown -R steam:steam /home/steam/steam/l4d2/left4dead2/addons diff --git a/bundles/l4d2/metadata.py b/bundles/l4d2/metadata.py new file mode 100644 index 0000000..d5294e3 --- /dev/null +++ b/bundles/l4d2/metadata.py @@ -0,0 +1,7 @@ +@metadata_reactor.provides() +def steam(metadata): + return { + 'steam': { + 222860: 'l4d2', + }, + } diff --git a/bundles/steam/files/steam-update.service b/bundles/steam/files/steam-update.service new file mode 100644 index 0000000..a91acd8 --- /dev/null +++ b/bundles/steam/files/steam-update.service @@ -0,0 +1,15 @@ +GNU nano 4.8 /etc/systemd/system/l4d2-update.service +[Unit] +Description=l4d2 update +After=network.target + +[Service] +Type=oneshot +User=steam +Group=steam +WorkingDirectory=/home/steam/steam +ExecStart=/home/steam/steam_workshop_downloader/workshop.py -o /home/steam/steam/l4d2/left4dead2/addons/workshop 2283884609 +ExecStart=/home/steam/steam/steamcmd.sh +login anonymous +force_install_dir ./l4d2/ +app_update 222860 validate +quit + +[Install] +WantedBy=multi-user.target diff --git a/bundles/steam/items.py b/bundles/steam/items.py new file mode 100644 index 0000000..83a52e7 --- /dev/null +++ b/bundles/steam/items.py @@ -0,0 +1,4 @@ +for id, name in node.metadata.get('steam'): + pass + +# sudo -Hiu steam bash -c '~/steam/steamcmd.sh +login anonymous +force_install_dir ./l4d2/ +app_update 222860 validate +quit' diff --git a/bundles/systemd/files/unitfile b/bundles/systemd/files/unitfile new file mode 100644 index 0000000..89d7d1b --- /dev/null +++ b/bundles/systemd/files/unitfile @@ -0,0 +1,19 @@ +% for i, (segment, options) in enumerate(data.items()): +% if i > 0: + +% endif +[${segment}] +% for option, value in options.items(): +% if isinstance(value, dict): +% for k, v in value.items(): +${option}=${k}=${v} +% endfor +% elif isinstance(value, (list, set, tuple)): +% for item in sorted(value): +${option}=${item} +% endfor +% elif isinstance(value, str): +${option}=${value} +% endif +% endfor +% endfor diff --git a/bundles/systemd/items.py b/bundles/systemd/items.py new file mode 100644 index 0000000..8bc32f2 --- /dev/null +++ b/bundles/systemd/items.py @@ -0,0 +1,52 @@ +timezone = node.metadata.get('timezone', 'UTC') +keymap = node.metadata.get('keymap', 'de') + +actions = { + 'systemd-reload': { + 'command': 'systemctl daemon-reload', + 'cascade_skip': False, + 'triggered': True, + 'needed_by': { + 'svc_systemd:', + }, + }, +} + +for name, service in node.metadata.get('systemd', {}).get('services', {}).items(): + # use set() in metadata + for enumerator in [ + 'preceded_by', 'needs', 'needed_by', 'triggers', 'triggered_by' + ]: + assert isinstance(service.get(enumerator, set()), set) + + # dont call a service 'service' explicitly + if name.endswith('.service'): + raise Exception(name) + + # split unit file content data from item data + content_data = service.pop('content') + + # default WantedBy=multi-user.target + content_data\ + .setdefault('Install', {})\ + .setdefault('WantedBy', {'multi-user.target'}) + + # create unit file + unit_path = f'/etc/systemd/system/{name}.service' + files[unit_path] = { + 'source': 'unitfile', + 'content_type': 'mako', + 'context': { + 'data': content_data, + }, + 'triggers': [ + 'action:systemd-reload', + f'svc_systemd:{name}:restart', + ], + } + + # service depends on unit file + service.setdefault('needs', set()).add(f'file:{unit_path}') + + # service + svc_systemd[name] = service diff --git a/groups.py b/groups.py new file mode 100644 index 0000000..6674fe8 --- /dev/null +++ b/groups.py @@ -0,0 +1,9 @@ +from os import walk +from os.path import join, basename, splitext + +for root, dirs, files in walk(join(repo_path, "groups")): + for filename in files: + if filename.endswith(".py"): + group = join(root, filename) + with open(group, 'r', encoding='utf-8') as f: + groups[splitext(basename(filename))[0]] = eval(f.read()) diff --git a/groups/os/debian-10.py b/groups/os/debian-10.py new file mode 100644 index 0000000..afa849b --- /dev/null +++ b/groups/os/debian-10.py @@ -0,0 +1,6 @@ +{ + 'supergroups': [ + 'debian', + ], + 'os_version': (10,) +} diff --git a/groups/os/debian.py b/groups/os/debian.py new file mode 100644 index 0000000..6f908af --- /dev/null +++ b/groups/os/debian.py @@ -0,0 +1,11 @@ +{ + 'supergroups': [ + 'linux', + ], + 'bundles': [ + 'apt', + 'systemd', + ], + 'os': 'debian', + 'pip_command': 'pip3', +} diff --git a/groups/os/linux.py b/groups/os/linux.py new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/groups/os/linux.py @@ -0,0 +1 @@ +{} diff --git a/items/download.py b/items/download.py new file mode 100644 index 0000000..c5eb3d5 --- /dev/null +++ b/items/download.py @@ -0,0 +1,120 @@ +from bundlewrap.items import Item, ItemStatus +from bundlewrap.exceptions import BundleError +from bundlewrap.utils.text import force_text, mark_for_translation as _ +from bundlewrap.utils.remote import PathInfo +import types +from pipes import quote + +# Downloaded from https://github.com/bundlewrap/plugins/blob/master/item_download/items/download.py +# No, we can't use plugins here, because bw4 won't support them anymore. + +class Download(Item): + """ + Download a file and verify its Hash. + """ + BUNDLE_ATTRIBUTE_NAME = "downloads" + NEEDS_STATIC = [ + "pkg_apt:", + "pkg_pacman:", + "pkg_yum:", + "pkg_zypper:", + ] + ITEM_ATTRIBUTES = { + 'url': "", + 'sha256': "", + 'verifySSL': True, + 'decompress': None, + } + ITEM_TYPE_NAME = "download" + REQUIRED_ATTRIBUTES = [] + + def __repr__(self): + return "".format(self.name) + + def __hash_remote_file(self, filename): + path_info = PathInfo(self.node, filename) + if not path_info.is_file: + return None + + if hasattr(path_info, 'sha256'): + return path_info.sha256 + else: + """"pending pr so do it manualy""" + if self.node.os == 'macos': + result = self.node.run("shasum -a 256 -- {}".format(quote(filename))) + elif self.node.os in self.node.OS_FAMILY_BSD: + result = self.node.run("sha256 -q -- {}".format(quote(filename))) + else: + result = self.node.run("sha256sum -- {}".format(quote(filename))) + return force_text(result.stdout).strip().split()[0] + + def fix(self, status): + if status.must_be_deleted: + # Not possible + pass + else: + decompress = self.attributes.get('decompress') + # download file + self.node.run("curl -L {verify}-s -- {url}{pipe} > {file}".format( + pipe = ' | ' + decompress if decompress else '', + verify="" if self.attributes.get('verifySSL', True) else "-k ", + file=quote(self.name), + url=quote(self.attributes['url']) + )) + + # check hash + sha256 = self.__hash_remote_file(self.name) + + if sha256 != self.attributes['sha256']: + # unlink file + self.node.run("rm -rf -- {}".format(quote(self.name))) + + return False + + def cdict(self): + """This is how the world should be""" + cdict = { + 'type': 'download', + 'sha256': self.attributes['sha256'], + } + + return cdict + + def sdict(self): + """This is how the world is right now""" + path_info = PathInfo(self.node, self.name) + if not path_info.exists: + return None + else: + sdict = { + 'type': 'download', + 'sha256': self.__hash_remote_file(self.name) + } + + return sdict + + @classmethod + def validate_attributes(cls, bundle, item_id, attributes): + if 'sha256' not in attributes: + raise BundleError(_( + "at least one hash must be set on {item} in bundle '{bundle}'" + ).format( + bundle=bundle.name, + item=item_id, + )) + + if 'url' not in attributes: + raise BundleError(_( + "you need to specify the url on {item} in bundle '{bundle}'" + ).format( + bundle=bundle.name, + item=item_id, + )) + + def get_auto_deps(self, items): + deps = [] + for item in items: + # debian TODO: add other package manager + if item.ITEM_TYPE_NAME == 'pkg_apt' and item.name == 'curl': + deps.append(item.id) + return deps diff --git a/nodes.py b/nodes.py new file mode 100644 index 0000000..8dc85bf --- /dev/null +++ b/nodes.py @@ -0,0 +1,9 @@ +from os import walk +from os.path import join, basename, splitext + +for root, dirs, files in walk(join(repo_path, "nodes")): + for filename in files: + if filename.endswith(".py"): + node = join(root, filename) + with open(node, 'r', encoding='utf-8') as f: + nodes[splitext(basename(filename))[0]] = eval(f.read()) diff --git a/nodes/backupserver.py b/nodes/backupserver.py new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/nodes/backupserver.py @@ -0,0 +1 @@ +{} diff --git a/nodes/gameserver.py b/nodes/gameserver.py new file mode 100644 index 0000000..11ccb41 --- /dev/null +++ b/nodes/gameserver.py @@ -0,0 +1,6 @@ +{ + 'bundles': [ + 'steam', + 'l4d2', + ], +} diff --git a/nodes/homeserver.py b/nodes/homeserver.py new file mode 100644 index 0000000..f88f7a9 --- /dev/null +++ b/nodes/homeserver.py @@ -0,0 +1,16 @@ +{ + 'hostname': '10.0.0.2', + 'bundles': [ + 'gitea', + ], + 'groups': [ + 'debian-10', + ], + 'metadata': { + 'gitea': { + 'version': '1.14.2', + 'sha256': '0d11d87ce60d5d98e22fc52f2c8c6ba2b54b14f9c26c767a46bf102c381ad128', + 'domain': 'git.sublimity.de', + }, + }, +} diff --git a/nodes/mailserver.py b/nodes/mailserver.py new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/nodes/mailserver.py @@ -0,0 +1 @@ +{} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..83c0c34 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +bundlewrap>=4.4.2