Compare commits
No commits in common. "1e62a44c16f8fe631f2b0e6fbc8174f3c3eea5e5" and "9985ecc56c476890daa45900a54fa4137dcc114a" have entirely different histories.
1e62a44c16
...
9985ecc56c
41 changed files with 1641 additions and 1801 deletions
|
|
@ -42,6 +42,10 @@ deploy/deploy-test-server.sh deploy-user@example-host
|
|||
|
||||
The SSH user must be able to run `sudo` on the target host. The deployment configures system packages, directories, environment files, helper scripts, sudoers rules, Python dependencies, and systemd units.
|
||||
|
||||
## Scheduled Jobs
|
||||
|
||||
`left4me-refresh-global-overlays.timer` runs daily with `Persistent=true`. It invokes `flask refresh-global-overlays`, which only enqueues a `refresh_global_overlays` job; downloads and rebuilds run in the web worker and are visible in the normal job log UI.
|
||||
|
||||
## Admin Bootstrap
|
||||
|
||||
Set the bootstrap credentials in the environment when creating the first admin user:
|
||||
|
|
@ -68,6 +72,6 @@ Invalid references are rejected:
|
|||
The web app currently supports two overlay surfaces:
|
||||
|
||||
- `workshop` overlays (user-owned) — populated by downloading `.vpk` files from the public Steam Web API into `${LEFT4ME_ROOT}/workshop_cache/{steam_id}.vpk` and creating absolute symlinks under `${LEFT4ME_ROOT}/overlays/{overlay_id}/left4dead2/addons/{steam_id}.vpk`.
|
||||
- `script` overlays — populated by an arbitrary user-authored bash script that runs inside `bubblewrap` + `systemd-run --scope` as the unprivileged `l4d2-sandbox` UID, with the overlay directory bind-mounted RW at `/overlay`. Resource caps: 1h walltime, 4 GB RAM, 512 tasks, 200% CPU, 20 GB post-build disk cap.
|
||||
- Managed global overlays (`l4d2center_maps`, `cedapug_maps`, system-wide) — populated by the daily `left4me-refresh-global-overlays` job, which downloads archives into `${LEFT4ME_ROOT}/global_overlay_cache/` and symlinks them into the overlay directory.
|
||||
|
||||
Both the caches and the overlay directories are owned by the `left4me` runtime user; if the web service ever runs as a different uid, ensure it shares a group with the host process and that both trees are group-readable.
|
||||
|
|
|
|||
|
|
@ -77,17 +77,11 @@ if ! id left4me >/dev/null 2>&1; then
|
|||
$sudo_cmd useradd --system --home-dir /var/lib/left4me --create-home --shell /usr/sbin/nologin left4me
|
||||
fi
|
||||
|
||||
# Sandbox uid for script-overlay builds. No home, no login shell — the bwrap
|
||||
# invocation uses --uid/--gid to drop to it.
|
||||
if ! id l4d2-sandbox >/dev/null 2>&1; then
|
||||
$sudo_cmd useradd --system --no-create-home --shell /usr/sbin/nologin l4d2-sandbox
|
||||
fi
|
||||
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
$sudo_cmd apt-get update
|
||||
$sudo_cmd apt-get install -y python3 python3-venv python3-pip curl ca-certificates tar gzip util-linux sudo bubblewrap
|
||||
$sudo_cmd apt-get install -y python3 python3-venv python3-pip curl ca-certificates tar gzip util-linux sudo
|
||||
elif command -v dnf >/dev/null 2>&1; then
|
||||
$sudo_cmd dnf install -y python3 python3-pip curl ca-certificates tar gzip util-linux sudo bubblewrap
|
||||
$sudo_cmd dnf install -y python3 python3-pip curl ca-certificates tar gzip util-linux sudo
|
||||
else
|
||||
printf 'Unsupported package manager: expected apt-get or dnf\n' >&2
|
||||
exit 1
|
||||
|
|
@ -103,6 +97,7 @@ $sudo_cmd mkdir -p \
|
|||
/var/lib/left4me/instances \
|
||||
/var/lib/left4me/runtime \
|
||||
/var/lib/left4me/workshop_cache \
|
||||
/var/lib/left4me/global_overlay_cache \
|
||||
/var/lib/left4me/tmp
|
||||
|
||||
$sudo_cmd chown left4me:left4me \
|
||||
|
|
@ -112,6 +107,7 @@ $sudo_cmd chown left4me:left4me \
|
|||
/var/lib/left4me/instances \
|
||||
/var/lib/left4me/runtime \
|
||||
/var/lib/left4me/workshop_cache \
|
||||
/var/lib/left4me/global_overlay_cache \
|
||||
/var/lib/left4me/tmp
|
||||
$sudo_cmd chown -R left4me:left4me /opt/left4me
|
||||
|
||||
|
|
@ -130,11 +126,12 @@ $sudo_cmd chown -R left4me:left4me /opt/left4me
|
|||
|
||||
$sudo_cmd cp /opt/left4me/deploy/files/usr/local/lib/systemd/system/left4me-web.service /usr/local/lib/systemd/system/left4me-web.service
|
||||
$sudo_cmd cp /opt/left4me/deploy/files/usr/local/lib/systemd/system/left4me-server@.service /usr/local/lib/systemd/system/left4me-server@.service
|
||||
$sudo_cmd cp /opt/left4me/deploy/files/usr/local/lib/systemd/system/left4me-refresh-global-overlays.service /usr/local/lib/systemd/system/left4me-refresh-global-overlays.service
|
||||
$sudo_cmd cp /opt/left4me/deploy/files/usr/local/lib/systemd/system/left4me-refresh-global-overlays.timer /usr/local/lib/systemd/system/left4me-refresh-global-overlays.timer
|
||||
$sudo_cmd cp /opt/left4me/deploy/files/usr/local/libexec/left4me/left4me-systemctl /usr/local/libexec/left4me/left4me-systemctl
|
||||
$sudo_cmd cp /opt/left4me/deploy/files/usr/local/libexec/left4me/left4me-journalctl /usr/local/libexec/left4me/left4me-journalctl
|
||||
$sudo_cmd cp /opt/left4me/deploy/files/usr/local/libexec/left4me/left4me-overlay /usr/local/libexec/left4me/left4me-overlay
|
||||
$sudo_cmd cp /opt/left4me/deploy/files/usr/local/libexec/left4me/left4me-script-sandbox /usr/local/libexec/left4me/left4me-script-sandbox
|
||||
$sudo_cmd chmod 0755 /usr/local/libexec/left4me/left4me-systemctl /usr/local/libexec/left4me/left4me-journalctl /usr/local/libexec/left4me/left4me-overlay /usr/local/libexec/left4me/left4me-script-sandbox
|
||||
$sudo_cmd chmod 0755 /usr/local/libexec/left4me/left4me-systemctl /usr/local/libexec/left4me/left4me-journalctl /usr/local/libexec/left4me/left4me-overlay
|
||||
$sudo_cmd cp /opt/left4me/deploy/files/etc/sudoers.d/left4me /etc/sudoers.d/left4me
|
||||
$sudo_cmd chmod 0440 /etc/sudoers.d/left4me
|
||||
$sudo_cmd visudo -cf /etc/sudoers.d/left4me
|
||||
|
|
@ -200,6 +197,7 @@ fi
|
|||
$sudo_cmd systemctl daemon-reload
|
||||
$sudo_cmd systemctl enable --now left4me-web.service
|
||||
$sudo_cmd systemctl restart left4me-web.service
|
||||
$sudo_cmd systemctl enable --now left4me-refresh-global-overlays.timer
|
||||
for attempt in 1 2 3 4 5 6 7 8 9 10; do
|
||||
if curl -fsS http://127.0.0.1:8000/health; then
|
||||
exit 0
|
||||
|
|
|
|||
|
|
@ -2,4 +2,3 @@ Defaults:left4me !requiretty
|
|||
left4me ALL=(root) NOPASSWD: /usr/local/libexec/left4me/left4me-systemctl *
|
||||
left4me ALL=(root) NOPASSWD: /usr/local/libexec/left4me/left4me-journalctl *
|
||||
left4me ALL=(root) NOPASSWD: /usr/local/libexec/left4me/left4me-overlay mount *, /usr/local/libexec/left4me/left4me-overlay umount *
|
||||
left4me ALL=(root) NOPASSWD: /usr/local/libexec/left4me/left4me-script-sandbox
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
[Unit]
|
||||
Description=left4me refresh global map overlays
|
||||
After=network-online.target left4me-web.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=left4me
|
||||
Group=left4me
|
||||
WorkingDirectory=/opt/left4me
|
||||
Environment=HOME=/var/lib/left4me
|
||||
Environment=PATH=/opt/left4me/.venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||
EnvironmentFile=/etc/left4me/host.env
|
||||
EnvironmentFile=/etc/left4me/web.env
|
||||
ExecStart=/opt/left4me/.venv/bin/flask --app l4d2web.app:create_app refresh-global-overlays
|
||||
ProtectSystem=full
|
||||
ReadWritePaths=/var/lib/left4me
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
[Unit]
|
||||
Description=Daily left4me global map overlay refresh
|
||||
|
||||
[Timer]
|
||||
OnCalendar=daily
|
||||
Persistent=true
|
||||
Unit=left4me-refresh-global-overlays.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
#!/bin/bash
|
||||
# Privileged sandbox launcher for left4me script overlays.
|
||||
#
|
||||
# Invoked via sudo by the web user with two arguments:
|
||||
# <overlay_id> numeric overlay id; bind-mounts /var/lib/left4me/overlays/<id>
|
||||
# read-write at /overlay inside the sandbox.
|
||||
# <script_path> absolute path to a bash file already written by the web app;
|
||||
# bind-mounted read-only at /script.sh inside the sandbox.
|
||||
#
|
||||
# The script runs under bubblewrap inside a transient systemd scope so we get
|
||||
# cgroup-v2 limits (memory / tasks / cpu) and a wallclock kill via
|
||||
# RuntimeMaxSec. The sandbox drops to the unprivileged l4d2-sandbox UID;
|
||||
# host filesystems are exposed read-only except /overlay (rw) and tmpfs
|
||||
# /tmp + /run. Network namespace is *not* unshared — scripts must reach the
|
||||
# public internet to download workshop / l4d2center / cedapug content.
|
||||
set -euo pipefail
|
||||
|
||||
[[ $# -eq 2 ]] || { echo "usage: $0 <overlay_id> <script>" >&2; exit 64; }
|
||||
|
||||
OVERLAY_ID=$1
|
||||
SCRIPT=$2
|
||||
|
||||
[[ "$OVERLAY_ID" =~ ^[0-9]+$ ]] || { echo "bad overlay id" >&2; exit 64; }
|
||||
OVERLAY_DIR=/var/lib/left4me/overlays/$OVERLAY_ID
|
||||
[[ -d $OVERLAY_DIR ]] || { echo "no overlay dir at $OVERLAY_DIR" >&2; exit 65; }
|
||||
[[ -f $SCRIPT ]] || { echo "no script at $SCRIPT" >&2; exit 65; }
|
||||
|
||||
SBX_UID=$(id -u l4d2-sandbox)
|
||||
SBX_GID=$(id -g l4d2-sandbox)
|
||||
|
||||
if [[ "${LEFT4ME_SCRIPT_SANDBOX_DRY_RUN:-}" == "1" ]]; then
|
||||
echo "DRY RUN: overlay_id=$OVERLAY_ID script=$SCRIPT uid=$SBX_UID gid=$SBX_GID overlay_dir=$OVERLAY_DIR"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
exec systemd-run --quiet --scope --collect \
|
||||
-p MemoryMax=4G -p MemorySwapMax=0 -p TasksMax=512 \
|
||||
-p CPUQuota=200% -p RuntimeMaxSec=3600 \
|
||||
-- bwrap \
|
||||
--die-with-parent --new-session \
|
||||
--unshare-pid --unshare-ipc --unshare-uts --unshare-cgroup \
|
||||
--uid "$SBX_UID" --gid "$SBX_GID" \
|
||||
--proc /proc --dev /dev --tmpfs /tmp --tmpfs /run \
|
||||
--ro-bind /usr /usr --ro-bind /lib /lib --ro-bind /lib64 /lib64 \
|
||||
--symlink usr/bin /bin --symlink usr/sbin /sbin \
|
||||
--ro-bind /etc/resolv.conf /etc/resolv.conf \
|
||||
--ro-bind /etc/ssl /etc/ssl \
|
||||
--ro-bind /etc/ca-certificates /etc/ca-certificates \
|
||||
--ro-bind /etc/nsswitch.conf /etc/nsswitch.conf \
|
||||
--bind "$OVERLAY_DIR" /overlay \
|
||||
--chdir /overlay \
|
||||
--setenv HOME /tmp --setenv PATH /usr/bin:/usr/sbin \
|
||||
--setenv OVERLAY /overlay \
|
||||
--ro-bind "$SCRIPT" /script.sh \
|
||||
/bin/bash /script.sh
|
||||
|
|
@ -11,11 +11,9 @@ WEB_UNIT = DEPLOY / "files/usr/local/lib/systemd/system/left4me-web.service"
|
|||
SERVER_UNIT = DEPLOY / "files/usr/local/lib/systemd/system/left4me-server@.service"
|
||||
GLOBAL_REFRESH_SERVICE = DEPLOY / "files/usr/local/lib/systemd/system/left4me-refresh-global-overlays.service"
|
||||
GLOBAL_REFRESH_TIMER = DEPLOY / "files/usr/local/lib/systemd/system/left4me-refresh-global-overlays.timer"
|
||||
SANDBOX_UNIT_DIR = DEPLOY / "files/usr/local/lib/systemd/system"
|
||||
SYSTEMCTL_HELPER = DEPLOY / "files/usr/local/libexec/left4me/left4me-systemctl"
|
||||
JOURNALCTL_HELPER = DEPLOY / "files/usr/local/libexec/left4me/left4me-journalctl"
|
||||
OVERLAY_HELPER = DEPLOY / "files/usr/local/libexec/left4me/left4me-overlay"
|
||||
SCRIPT_SANDBOX_HELPER = DEPLOY / "files/usr/local/libexec/left4me/left4me-script-sandbox"
|
||||
SUDOERS = DEPLOY / "files/etc/sudoers.d/left4me"
|
||||
HOST_ENV = DEPLOY / "templates/etc/left4me/host.env"
|
||||
WEB_ENV_TEMPLATE = DEPLOY / "templates/etc/left4me/web.env.template"
|
||||
|
|
@ -157,10 +155,6 @@ def test_sudoers_allows_only_left4me_helpers_not_raw_system_tools():
|
|||
) in sudoers
|
||||
assert "/usr/local/libexec/left4me/left4me-overlay mount *" in sudoers
|
||||
assert "/usr/local/libexec/left4me/left4me-overlay umount *" in sudoers
|
||||
assert (
|
||||
"left4me ALL=(root) NOPASSWD: "
|
||||
"/usr/local/libexec/left4me/left4me-script-sandbox"
|
||||
) in sudoers
|
||||
assert "/bin/systemctl" not in sudoers
|
||||
assert "/usr/bin/systemctl" not in sudoers
|
||||
assert "/bin/journalctl" not in sudoers
|
||||
|
|
@ -271,87 +265,23 @@ def test_deploy_script_shell_syntax() -> None:
|
|||
subprocess.run(["sh", "-n", str(DEPLOY_SCRIPT)], check=True)
|
||||
|
||||
|
||||
def test_globals_refresh_units_removed():
|
||||
"""Global-overlays subsystem deleted in favor of script overlays."""
|
||||
assert not GLOBAL_REFRESH_SERVICE.exists()
|
||||
assert not GLOBAL_REFRESH_TIMER.exists()
|
||||
def test_global_refresh_timer_units_exist_and_enqueue_only():
|
||||
service = GLOBAL_REFRESH_SERVICE.read_text()
|
||||
timer = GLOBAL_REFRESH_TIMER.read_text()
|
||||
|
||||
assert "User=left4me" in service
|
||||
assert "EnvironmentFile=/etc/left4me/host.env" in service
|
||||
assert "EnvironmentFile=/etc/left4me/web.env" in service
|
||||
assert "flask --app l4d2web.app:create_app refresh-global-overlays" in service
|
||||
assert "OnCalendar=daily" in timer
|
||||
assert "Persistent=true" in timer
|
||||
assert "WantedBy=timers.target" in timer
|
||||
|
||||
|
||||
def test_deploy_script_does_not_reference_globals_subsystem():
|
||||
def test_deploy_script_installs_and_enables_global_refresh_timer():
|
||||
script = DEPLOY_SCRIPT.read_text()
|
||||
|
||||
assert "/var/lib/left4me/global_overlay_cache" not in script
|
||||
assert "left4me-refresh-global-overlays" not in script
|
||||
|
||||
|
||||
def test_deploy_script_provisions_sandbox_user():
|
||||
script = DEPLOY_SCRIPT.read_text()
|
||||
assert "useradd --system --no-create-home --shell /usr/sbin/nologin l4d2-sandbox" in script
|
||||
|
||||
|
||||
def test_deploy_script_installs_bubblewrap():
|
||||
install_lines = [
|
||||
line for line in DEPLOY_SCRIPT.read_text().splitlines()
|
||||
if ("apt-get install" in line or "dnf install" in line)
|
||||
]
|
||||
assert install_lines, "expected at least one apt/dnf install line"
|
||||
for line in install_lines:
|
||||
assert "bubblewrap" in line, f"missing bubblewrap in install line: {line}"
|
||||
|
||||
|
||||
def test_deploy_script_installs_script_sandbox_helper():
|
||||
script = DEPLOY_SCRIPT.read_text()
|
||||
assert "/usr/local/libexec/left4me/left4me-script-sandbox" in script
|
||||
assert "chmod 0755" in script and "left4me-script-sandbox" in script
|
||||
|
||||
|
||||
def test_script_sandbox_helper_present():
|
||||
assert SCRIPT_SANDBOX_HELPER.is_file()
|
||||
assert SCRIPT_SANDBOX_HELPER.read_text().startswith("#!/bin/bash")
|
||||
mode = SCRIPT_SANDBOX_HELPER.stat().st_mode & 0o777
|
||||
assert mode == 0o755, f"expected 0755, got {oct(mode)}"
|
||||
|
||||
|
||||
def test_script_sandbox_helper_passes_shell_syntax_check():
|
||||
subprocess.run(["bash", "-n", str(SCRIPT_SANDBOX_HELPER)], check=True)
|
||||
|
||||
|
||||
def test_script_sandbox_helper_invokes_systemd_run_and_bwrap():
|
||||
text = SCRIPT_SANDBOX_HELPER.read_text()
|
||||
assert "systemd-run" in text
|
||||
assert "--scope" in text
|
||||
assert "--collect" in text
|
||||
assert "MemoryMax=4G" in text
|
||||
assert "RuntimeMaxSec=3600" in text
|
||||
assert "TasksMax=512" in text
|
||||
assert "bwrap" in text
|
||||
assert "--unshare-pid" in text
|
||||
assert "--unshare-net" not in text, "scripts must keep host network access"
|
||||
assert 'id -u l4d2-sandbox' in text
|
||||
assert 'id -g l4d2-sandbox' in text
|
||||
|
||||
|
||||
def test_script_sandbox_helper_validates_overlay_id():
|
||||
text = SCRIPT_SANDBOX_HELPER.read_text()
|
||||
# Numeric-only overlay id
|
||||
assert '[[ "$OVERLAY_ID" =~ ^[0-9]+$ ]]' in text
|
||||
# Overlay dir must exist
|
||||
assert "/var/lib/left4me/overlays/" in text
|
||||
assert "[[ -d $OVERLAY_DIR ]]" in text
|
||||
# Script path must exist
|
||||
assert "[[ -f $SCRIPT ]]" in text
|
||||
|
||||
|
||||
def test_script_sandbox_helper_dry_run_mode(tmp_path):
|
||||
overlay_root = tmp_path / "var/lib/left4me/overlays/42"
|
||||
overlay_root.mkdir(parents=True)
|
||||
fake_script = tmp_path / "fake.sh"
|
||||
fake_script.write_text("echo hi")
|
||||
|
||||
# Run in DRY_RUN mode against a fake l4d2-sandbox UID via a tiny shim that
|
||||
# simulates `id -u l4d2-sandbox` resolving to a valid number.
|
||||
helper_text = SCRIPT_SANDBOX_HELPER.read_text()
|
||||
# We can't actually exec this without root + a real sandbox user; just
|
||||
# verify the dry-run guard short-circuits before systemd-run / bwrap.
|
||||
assert 'LEFT4ME_SCRIPT_SANDBOX_DRY_RUN' in helper_text
|
||||
assert 'exit 0' in helper_text
|
||||
assert "/var/lib/left4me/global_overlay_cache" in script
|
||||
assert "left4me-refresh-global-overlays.service" in script
|
||||
assert "left4me-refresh-global-overlays.timer" in script
|
||||
assert "systemctl enable --now left4me-refresh-global-overlays.timer" in script
|
||||
|
|
|
|||
|
|
@ -1,350 +0,0 @@
|
|||
# L4D2 Script Overlays Implementation Plan
|
||||
|
||||
> **Approval status:** User-approved 2026-05-08. Implementation proceeds.
|
||||
|
||||
**Goal:** Implement the `script` overlay type per `docs/superpowers/specs/2026-05-08-l4d2-script-overlays-design.md`. Add an `Overlay.script` TEXT column and `Overlay.last_build_status` enum-string column, a `ScriptBuilder` that runs user bash inside a `bubblewrap` + `systemd-run --scope` sandbox via a new `left4me-script-sandbox` privileged helper, route + UI surface for editing/wiping/rebuilding, and delete the entire managed-globals (`l4d2center_maps`, `cedapug_maps`) subsystem and its daily-refresh timer/CLI.
|
||||
|
||||
**Architecture:** The web app continues to enqueue `build_overlay` jobs for any overlay row. The job worker dispatches via `BUILDERS[overlay.type].build(...)`. After this change `BUILDERS = {"workshop": WorkshopBuilder(), "script": ScriptBuilder()}`. The new `ScriptBuilder` writes `overlay.script` to a tmpfile and execs `sudo -n /usr/local/libexec/left4me/left4me-script-sandbox <id> <tmpfile>`, which itself execs `systemd-run --scope --collect ... -- bwrap [namespace flags] /bin/bash /script.sh`. stdout/stderr stream through the existing `run_with_streamed_output` helper into the existing job-log SSE plumbing. The job-completion path writes `Overlay.last_build_status` based on the build outcome. The kernel-overlayfs mount layer (`KernelOverlayFSMounter`) is unchanged.
|
||||
|
||||
---
|
||||
|
||||
## Locked Decisions
|
||||
|
||||
See `docs/superpowers/specs/2026-05-08-l4d2-script-overlays-design.md` for design rationale. Implementation-relevant summary:
|
||||
|
||||
- Final overlay type list: `workshop` (unchanged) + `script` (new). Drop `l4d2center_maps`, `cedapug_maps`.
|
||||
- New columns on `overlays`: `script TEXT NOT NULL DEFAULT ''`, `last_build_status VARCHAR(16) NOT NULL DEFAULT ''`.
|
||||
- Drop tables (FK order): `global_overlay_item_files`, `global_overlay_items`, `global_overlay_sources`.
|
||||
- `ScriptBuilder` in `l4d2web/services/overlay_builders.py`, uses existing `run_with_streamed_output`.
|
||||
- Privileged helper `left4me-script-sandbox` (bash, mode 0755, owned root). `systemd-run --scope --collect -p MemoryMax=4G -p MemorySwapMax=0 -p TasksMax=512 -p CPUQuota=200% -p RuntimeMaxSec=3600 -- bwrap …`. Limits 1 h walltime, 4 GB RAM, 20 GB post-build `du` cap.
|
||||
- New system user `l4d2-sandbox` (`/usr/sbin/nologin`, no home). New apt dep `bubblewrap`.
|
||||
- Sudoers verb-unrestricted: `left4me ALL=(root) NOPASSWD: /usr/local/libexec/left4me/left4me-script-sandbox`.
|
||||
- Daily refresh subsystem deleted: `left4me-refresh-global-overlays.{timer,service}` and `flask refresh-global-overlays` CLI removed. No replacement.
|
||||
- Wipe is the same sandbox helper invoked with the literal script `find /overlay -mindepth 1 -delete`.
|
||||
- `auto_refresh` column NOT added in this iteration.
|
||||
- Test deploy DB is wiped on rollout; migration includes `DELETE FROM overlays WHERE type IN ('l4d2center_maps', 'cedapug_maps')` for safety.
|
||||
|
||||
---
|
||||
|
||||
## Current Gap
|
||||
|
||||
- `l4d2web/models.py` `Overlay` has no `script` or `last_build_status` columns. The 3 globals tables are present.
|
||||
- `l4d2web/services/overlay_builders.py` `BUILDERS = {"workshop": WorkshopBuilder(), "l4d2center_maps": GlobalMapOverlayBuilder(), "cedapug_maps": GlobalMapOverlayBuilder()}`. No `ScriptBuilder`.
|
||||
- `l4d2web/services/{global_map_sources,global_overlay_refresh,global_map_cache,global_overlays}.py` exist and are referenced by routes / CLI.
|
||||
- `l4d2web/services/job_worker.py` carries `refresh_global_overlays_running` plumbing.
|
||||
- `l4d2web/cli.py` defines `refresh-global-overlays`.
|
||||
- `l4d2web/routes/overlay_routes.py` has no `/script`, `/wipe`, or `/build` endpoints for non-workshop types.
|
||||
- `l4d2web/templates/overlays.html` create modal type radio offers only `workshop`.
|
||||
- `l4d2web/templates/overlay_detail.html` has a global-source block (~lines 34–46) that should not survive.
|
||||
- `deploy/files/usr/local/lib/systemd/system/left4me-refresh-global-overlays.{timer,service}` exist.
|
||||
- `deploy/deploy-test-server.sh` provisions `global_overlay_cache/` and does not provision `l4d2-sandbox` or install `bubblewrap`.
|
||||
- Seven `tests/test_global_*.py` files exist and reference removed code.
|
||||
|
||||
---
|
||||
|
||||
## Task 1: Schema migration (alembic 0005)
|
||||
|
||||
**Files:**
|
||||
|
||||
- Create: `l4d2web/alembic/versions/0005_script_overlays.py` (revises `0004_drop_legacy_external_overlay_type`).
|
||||
- Modify: `l4d2web/models.py` — `Overlay` gains `script` and `last_build_status` columns; remove `GlobalOverlaySource`, `GlobalOverlayItem`, `GlobalOverlayItemFile` model classes.
|
||||
- Modify: `l4d2web/tests/test_overlay_models.py` (or whichever existing test asserts the Overlay schema; create one if absent) — assert new columns present.
|
||||
|
||||
Test plan (RED first):
|
||||
|
||||
1. `tests/test_alembic_migrations.py::test_upgrade_0005_adds_script_columns` — apply migrations to a fresh in-memory SQLite, assert `script` and `last_build_status` columns present on `overlays`, assert no `global_overlay_*` tables, assert old data wipe `DELETE FROM overlays WHERE type IN (...)` is part of the upgrade.
|
||||
2. `tests/test_alembic_migrations.py::test_downgrade_0005_restores_globals` (only if downgrade is supported in the project's migration policy; skip with `pytest.skip` if not — kernel-overlayfs migration is one-way, follow that precedent).
|
||||
3. `tests/test_overlay_models.py::test_overlay_has_script_columns` — `Overlay(...)` instance has `script=''` and `last_build_status=''` defaults.
|
||||
|
||||
Implementation:
|
||||
|
||||
- Migration uses `op.drop_table('global_overlay_item_files')` etc. in correct FK order; uses `op.add_column('overlays', sa.Column('script', sa.Text(), nullable=False, server_default=''))` and similar for `last_build_status` (`sa.String(16)`).
|
||||
- The `DELETE FROM overlays WHERE type IN ('l4d2center_maps','cedapug_maps')` runs *before* the column additions so the operation is straightforward — these rows do not reference the new columns.
|
||||
- `models.py`: delete the three globals model classes outright; add the two new columns to `Overlay` with explicit defaults.
|
||||
|
||||
**Verification:**
|
||||
|
||||
```
|
||||
python3 -m pytest l4d2web/tests/test_alembic_migrations.py l4d2web/tests/test_overlay_models.py -q
|
||||
```
|
||||
|
||||
**Commit:** `feat(l4d2-web): script overlay schema — add overlay.script + last_build_status, drop globals tables`
|
||||
|
||||
---
|
||||
|
||||
## Task 2: ScriptBuilder + BUILDERS registry update
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `l4d2web/services/overlay_builders.py` — add `ScriptBuilder`, remove `GlobalMapOverlayBuilder`, change `BUILDERS` dict.
|
||||
- Rewrite: `l4d2web/tests/test_overlay_builders.py` — drop globals-builder tests, add ScriptBuilder tests.
|
||||
|
||||
Test plan (RED first):
|
||||
|
||||
1. `test_overlay_builders.py::test_builders_registry` — `set(BUILDERS) == {"workshop", "script"}`. Assert `"l4d2center_maps"` and `"cedapug_maps"` and `"external"` are absent.
|
||||
2. `test_overlay_builders.py::test_script_builder_invokes_helper` — patch `run_with_streamed_output` to capture argv; build an `Overlay(id=42, type='script', script='echo hi')`; assert argv shape `["sudo", "-n", "/usr/local/libexec/left4me/left4me-script-sandbox", "42", <script_path>]` and that the script_path file exists with content `"echo hi"` at invocation time. Verify the tmpfile is unlinked after build.
|
||||
3. `test_overlay_builders.py::test_script_builder_disk_cap` — fake `subprocess.check_output` for `du` to return `25000000000`; build raises `BuildError("disk-cap-exceeded")` and `on_stderr` was called with the cap message.
|
||||
4. `test_overlay_builders.py::test_script_builder_streams_output` — fake `run_with_streamed_output` invokes both `on_stdout("hello\n")` and `on_stderr("warn\n")`; both lambda lists capture the lines.
|
||||
5. `test_overlay_builders.py::test_script_builder_cancel` — `should_cancel` returns True after the first stdout line; assert `run_with_streamed_output` propagated cancellation (the existing helper's contract — the test just ensures we pass `should_cancel` through and don't run the disk-budget check on cancel).
|
||||
6. `test_overlay_builders.py::test_workshop_builder_unchanged` — smoke test that `WorkshopBuilder` still exists and is invokable (regression guard against accidental removal during refactor).
|
||||
|
||||
Implementation:
|
||||
|
||||
- Add `import os, subprocess, tempfile` at the top of `overlay_builders.py` if not present.
|
||||
- `ScriptBuilder` exactly as in the spec (verbatim copy from the design doc, §Build Lifecycle).
|
||||
- Define a small `BuildError` exception class if one doesn't already exist locally; reuse the existing one if `WorkshopBuilder` already raises a similar type.
|
||||
- `_enforce_disk_budget` calls `subprocess.check_output(["du", "-sb", str(overlay_path(overlay_id))])`; the existing `overlay_path` helper in the module already returns the absolute Path. Parse first whitespace-delimited integer; cap is `20 * 1024**3`.
|
||||
- Job-completion path: locate the existing path that handles `build_overlay` job success/failure (likely in `services/job_worker.py` or a related orchestration module). Add a single column write: on success `last_build_status='ok'`, on `BuildError` / non-zero exit / cancel `last_build_status='failed'`. Add a `tests/test_job_worker.py::test_build_overlay_writes_last_build_status` covering both branches.
|
||||
- Remove `GlobalMapOverlayBuilder` class and any helper functions it owns that are not used elsewhere.
|
||||
|
||||
**Verification:**
|
||||
|
||||
```
|
||||
python3 -m pytest l4d2web/tests/test_overlay_builders.py l4d2web/tests/test_job_worker.py -q
|
||||
```
|
||||
|
||||
**Commit:** `feat(l4d2-web): ScriptBuilder + BUILDERS registry update`
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Delete global-overlay services + CLI command + their tests
|
||||
|
||||
**Files:**
|
||||
|
||||
- Delete: `l4d2web/services/global_map_sources.py`
|
||||
- Delete: `l4d2web/services/global_overlay_refresh.py`
|
||||
- Delete: `l4d2web/services/global_map_cache.py`
|
||||
- Delete: `l4d2web/services/global_overlays.py`
|
||||
- Modify: `l4d2web/cli.py` — remove `refresh-global-overlays` command (lines ~44–55). Drop any imports that go orphaned.
|
||||
- Delete: `l4d2web/tests/test_global_map_sources.py`
|
||||
- Delete: `l4d2web/tests/test_global_overlay_models.py`
|
||||
- Delete: `l4d2web/tests/test_global_overlay_builders.py`
|
||||
- Delete: `l4d2web/tests/test_global_overlay_cli.py`
|
||||
- Delete: `l4d2web/tests/test_global_overlay_refresh.py`
|
||||
- Delete: `l4d2web/tests/test_global_overlays.py`
|
||||
- Delete: `l4d2web/tests/test_global_map_cache.py`
|
||||
- Audit & fix: any other module that imports the deleted modules. Likely candidates: `l4d2web/app.py` (CLI registration), `routes/overlay_routes.py`, `routes/page_routes.py`. Resolve by deletion of the dead import / call site, not by stubbing.
|
||||
- Modify: `pyproject.toml` — drop `py7zr` from dependencies (only used by the deleted globals subsystem).
|
||||
|
||||
Test plan:
|
||||
|
||||
1. RED-first via grep: `grep -RIn 'global_map_sources\|global_overlay_refresh\|global_map_cache\|global_overlays\|refresh_global_overlays\|GlobalMapOverlayBuilder' l4d2web/ deploy/` — should return zero hits at the end of this task. Add this as `tests/test_no_globals_references.py::test_no_globals_imports` if you want it as a permanent regression guard, otherwise spot-check.
|
||||
2. Existing `tests/test_cli.py` (or whichever covers Flask CLI) loses any cases for `refresh-global-overlays`; add a `test_refresh_global_overlays_command_removed` that asserts the click command is not registered.
|
||||
|
||||
Implementation:
|
||||
|
||||
- Delete files via `git rm`.
|
||||
- In `cli.py`, remove the command function and its `@app.cli.command(...)` decorator. Drop any helper imports that become orphaned.
|
||||
- Remove `py7zr` from `pyproject.toml` and re-lock if a lockfile is present.
|
||||
|
||||
**Verification:**
|
||||
|
||||
```
|
||||
python3 -m pytest l4d2web/tests/ -q
|
||||
grep -RIn 'global_map_sources\|global_overlay_refresh\|global_map_cache\|global_overlays\|refresh_global_overlays\|GlobalMapOverlayBuilder' l4d2web/ deploy/ || echo "clean"
|
||||
```
|
||||
|
||||
**Commit:** `refactor(l4d2-web): drop global-overlays subsystem in favor of script type`
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Job worker — drop refresh_global_overlays from scheduler
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `l4d2web/services/job_worker.py` — remove `"refresh_global_overlays"` from `GLOBAL_OPERATIONS`; remove `refresh_global_overlays_running` field from `SchedulerState` and any references in `can_start()`; check whether `blocked_servers_by_overlay` was added solely for the globals subsystem and remove if so.
|
||||
- Modify: `l4d2web/tests/test_job_worker.py` — drop `refresh_global_overlays` truth-table rows; add explicit `build_overlay` truth-table cases for `script`-type overlays (mechanically identical to workshop, but pinned by test).
|
||||
|
||||
Test plan:
|
||||
|
||||
1. `test_job_worker.py::test_global_operations_set` — `GLOBAL_OPERATIONS == {"install", "refresh_workshop_items"}` (or whatever subset remains; pin it).
|
||||
2. `test_job_worker.py::test_build_overlay_script_type_blocks_per_overlay` — start `build_overlay(overlay_id=7)` for a `script`-type overlay; assert second `build_overlay(overlay_id=7)` cannot start; assert `build_overlay(overlay_id=8)` can.
|
||||
3. `test_job_worker.py::test_build_overlay_blocks_server_init_on_blueprint_overlay` — existing test, may need re-pinning if it referenced globals.
|
||||
|
||||
Implementation:
|
||||
|
||||
- Remove the field from the dataclass / TypedDict that backs `SchedulerState`.
|
||||
- Remove any update sites that flipped the flag (the worker's enqueue / on-start / on-complete paths).
|
||||
- The remaining mutex rules (`install` / `refresh_workshop_items` are global; `build_overlay` per-overlay; server ops block on overlays in their blueprint) are unchanged structurally.
|
||||
|
||||
**Verification:**
|
||||
|
||||
```
|
||||
python3 -m pytest l4d2web/tests/test_job_worker.py -q
|
||||
```
|
||||
|
||||
**Commit:** `refactor(l4d2-web): drop refresh_global_overlays from scheduler`
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Routes (script update / wipe / build)
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `l4d2web/routes/overlay_routes.py` — add three POST endpoints.
|
||||
- Create: `l4d2web/tests/test_script_overlay_routes.py`.
|
||||
|
||||
Test plan (RED first):
|
||||
|
||||
1. `test_script_overlay_routes.py::test_create_script_overlay` — POST `/overlays` with form `{"name": "x", "type": "script"}` as a regular user → 302 to detail; row exists with `type='script'`, `script=''`, `last_build_status=''`, `user_id=current_user.id`, `path=str(id)`.
|
||||
2. `test_script_overlay_routes.py::test_admin_creates_system_wide_script_overlay` — admin POST with system-wide flag → row has `user_id=NULL`.
|
||||
3. `test_script_overlay_routes.py::test_update_script_body_enqueues_build` — POST `/overlays/{id}/script` with `{"script": "echo new"}` → row.script updated; one new `build_overlay` job enqueued for the overlay; second immediate POST coalesces (no second job inserted while first is pending).
|
||||
4. `test_script_overlay_routes.py::test_manual_rebuild` — POST `/overlays/{id}/build` → enqueues `build_overlay`; coalesces.
|
||||
5. `test_script_overlay_routes.py::test_wipe_runs_find_delete` — POST `/overlays/{id}/wipe` → invokes `ScriptBuilder.build` (or the underlying helper) with the literal script `find /overlay -mindepth 1 -delete`. After success, row.last_build_status `==''`. Does not enqueue a `build_overlay`.
|
||||
6. `test_script_overlay_routes.py::test_wipe_refuses_during_running_build` — set scheduler state to `build_overlay(overlay_id=7)` running; POST `/overlays/7/wipe` → 409 (or whatever the existing pattern uses for scheduler conflicts), no sandbox invocation.
|
||||
7. `test_script_overlay_routes.py::test_permissions_non_owner_denied` — user A creates private script overlay; user B POSTs `/overlays/{id}/script` → 403.
|
||||
8. `test_script_overlay_routes.py::test_permissions_admin_can_edit_any` — admin POSTs `/overlays/{id}/script` for user A's row → 200.
|
||||
|
||||
Implementation:
|
||||
|
||||
- Mirror the existing `_can_edit_overlay()` permission helper.
|
||||
- The `/wipe` endpoint can either (a) call `ScriptBuilder` directly with a synthetic `Overlay`-like object whose `.script` is the find command and whose `.id` is the real overlay id, or (b) factor a `_run_sandbox(overlay_id, script_text, on_stdout, on_stderr, should_cancel)` helper out of `ScriptBuilder.build()` and call it from both. (b) is cleaner; do (b).
|
||||
- Wipe runs **synchronously** in the request thread (small, fast). It does NOT enqueue a job. Surface log output as flash messages or by streaming through the existing log infra — pick whichever matches the existing wipe-equivalent pattern (workshop overlays don't have a wipe; closest analog is the existing delete-overlay flow).
|
||||
- The `/script` endpoint enqueues via the same `enqueue_build_overlay(overlay_id)` helper used by workshop overlays' add/remove flows. Coalescing is already implemented there.
|
||||
|
||||
**Verification:**
|
||||
|
||||
```
|
||||
python3 -m pytest l4d2web/tests/test_script_overlay_routes.py l4d2web/tests/test_overlay_routes.py -q
|
||||
```
|
||||
|
||||
**Commit:** `feat(l4d2-web): script overlay routes (script update / wipe / build)`
|
||||
|
||||
---
|
||||
|
||||
## Task 6: Templates (overlays.html + overlay_detail.html)
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `l4d2web/templates/overlays.html` — add `script` to the create-modal type radio (lines ~29–49).
|
||||
- Modify: `l4d2web/templates/overlay_detail.html` — add a `{% if overlay.type == 'script' %}` block with textarea + Save / Rebuild / Wipe buttons + status badge; delete the global-source block (lines ~34–46).
|
||||
- Modify: `l4d2web/tests/test_pages.py` — assert script-section renders for type=`script`, workshop-section renders for type=`workshop`, global-source-section is absent.
|
||||
|
||||
Test plan:
|
||||
|
||||
1. `test_pages.py::test_overlay_create_modal_offers_script_type` — GET `/overlays`; HTML contains `value="script"` radio.
|
||||
2. `test_pages.py::test_overlay_detail_script_section` — create script overlay, GET `/overlays/{id}`; HTML contains `<textarea name="script">`, "Rebuild" button, "Wipe" button, status badge element.
|
||||
3. `test_pages.py::test_overlay_detail_workshop_section_unchanged` — existing workshop detail still has thumbnail grid, add-item form, etc.
|
||||
4. `test_pages.py::test_overlay_detail_no_global_source_block` — page HTML has no element from the deleted global-source block (check for an attribute or string unique to that block).
|
||||
|
||||
Implementation:
|
||||
|
||||
- Detail-page wipe button uses a small confirm-modal pattern (copy from the existing delete-overlay confirm modal).
|
||||
- Status badge: existing CSS classes for ok/warn/error already exist in `static/`; reuse them.
|
||||
- No new JS deps. Plain `<form method="post">` with HTMX `hx-post` for the script update if a streaming UX is desired (match existing patterns).
|
||||
|
||||
**Verification:**
|
||||
|
||||
```
|
||||
python3 -m pytest l4d2web/tests/test_pages.py -q
|
||||
```
|
||||
|
||||
Manual: start dev server (`flask run`), create a script overlay, paste `echo "hi" > foo`, click Save, watch log stream. Then click Wipe; confirm dir is empty. Then click Rebuild; confirm `foo` reappears.
|
||||
|
||||
**Commit:** `feat(l4d2-web): script overlay UI`
|
||||
|
||||
---
|
||||
|
||||
## Task 7: Libexec sandbox helper + sudoers + deploy-artifacts test
|
||||
|
||||
**Files:**
|
||||
|
||||
- Create: `deploy/files/usr/local/libexec/left4me/left4me-script-sandbox` (bash, mode 0755 after deploy, owned root).
|
||||
- Modify: `deploy/files/etc/sudoers.d/left4me` — append the rule.
|
||||
- Modify: `deploy/tests/test_deploy_artifacts.py` — assert helper file present + sudoers contains the new line.
|
||||
|
||||
Test plan (RED first):
|
||||
|
||||
1. `test_deploy_artifacts.py::test_script_sandbox_helper_present` — file exists, mode bits indicate 0755 (or whatever the test framework allows checking pre-deploy), shebang is `#!/bin/bash`.
|
||||
2. `test_deploy_artifacts.py::test_sudoers_includes_script_sandbox_rule` — sudoers file contains the exact line `left4me ALL=(root) NOPASSWD: /usr/local/libexec/left4me/left4me-script-sandbox`.
|
||||
3. Optional integration test (skip on non-Linux dev): drive the helper as a subprocess with a synthesized fake `/var/lib/left4me/overlays/1/` and a no-op script, assert `bwrap` invocation happens (use a mock `systemd-run` or `LEFT4ME_SCRIPT_SANDBOX_DRY_RUN=1` env that prints the would-be invocation and exits 0). Mirrors the `LEFT4ME_OVERLAY_PRINT_ONLY=1` pattern from the kernel-overlayfs helper test.
|
||||
|
||||
Implementation:
|
||||
|
||||
- Helper script verbatim from the spec §Sandbox.
|
||||
- Sudoers fragment: append (don't replace existing rules). The existing fragment has rules for `left4me-overlay`, `left4me-systemctl`, `left4me-journalctl` — match the same formatting (one rule per line, no trailing whitespace).
|
||||
|
||||
**Verification:**
|
||||
|
||||
```
|
||||
python3 -m pytest deploy/tests/test_deploy_artifacts.py -q
|
||||
bash -n deploy/files/usr/local/libexec/left4me/left4me-script-sandbox
|
||||
```
|
||||
|
||||
**Commit:** `feat(deploy): left4me-script-sandbox helper + sudoers fragment`
|
||||
|
||||
---
|
||||
|
||||
## Task 8: Deploy script — provision l4d2-sandbox + bubblewrap; drop globals timer
|
||||
|
||||
**Files:**
|
||||
|
||||
- Modify: `deploy/deploy-test-server.sh` — add `useradd --system ... l4d2-sandbox`, add `apt-get install -y bubblewrap`, ensure helper installation step picks up `left4me-script-sandbox` (likely automatic if it's a glob in `deploy/files/usr/local/libexec/left4me/*`); drop the `mkdir global_overlay_cache` line if present.
|
||||
- Delete: `deploy/files/usr/local/lib/systemd/system/left4me-refresh-global-overlays.timer`
|
||||
- Delete: `deploy/files/usr/local/lib/systemd/system/left4me-refresh-global-overlays.service`
|
||||
- Modify: `deploy/tests/test_deploy_artifacts.py` — assert the two unit files are absent; assert `useradd l4d2-sandbox` and `apt-get install ... bubblewrap` lines are present in the deploy script.
|
||||
|
||||
Test plan:
|
||||
|
||||
1. `test_deploy_artifacts.py::test_globals_refresh_units_removed` — files do not exist under `deploy/files/usr/local/lib/systemd/system/`.
|
||||
2. `test_deploy_artifacts.py::test_deploy_script_provisions_sandbox_user` — grep the deploy script for the useradd line.
|
||||
3. `test_deploy_artifacts.py::test_deploy_script_installs_bubblewrap` — grep for `bubblewrap` in apt invocations.
|
||||
|
||||
Implementation:
|
||||
|
||||
- `useradd` line uses `--system --no-create-home --shell /usr/sbin/nologin`. Idempotency: wrap with `id l4d2-sandbox &>/dev/null || useradd ...`.
|
||||
- `apt-get install`: append `bubblewrap` to whatever package list the script already maintains.
|
||||
- Globals timer/service deletions: `git rm`.
|
||||
|
||||
**Verification:**
|
||||
|
||||
```
|
||||
python3 -m pytest deploy/tests/ -q
|
||||
shellcheck deploy/deploy-test-server.sh deploy/files/usr/local/libexec/left4me/left4me-script-sandbox
|
||||
```
|
||||
|
||||
**Commit:** `chore(deploy): provision l4d2-sandbox + bubblewrap; drop globals refresh timer`
|
||||
|
||||
---
|
||||
|
||||
## Task 9: Full pytest run + drift fixes
|
||||
|
||||
**Files:** as needed across the repo.
|
||||
|
||||
Test plan: run the full test suite for both packages; chase down any drift caused by removed model classes, dropped imports, or template changes.
|
||||
|
||||
```
|
||||
python3 -m pytest l4d2web/tests/ -q
|
||||
python3 -m pytest l4d2host/tests/ -q
|
||||
python3 -m pytest deploy/tests/ -q
|
||||
```
|
||||
|
||||
Implementation: fix what breaks. Common drift sources to expect:
|
||||
|
||||
- Tests that imported from deleted modules.
|
||||
- Tests that asserted exact `BUILDERS` keyset (good — they should have been updated in Task 2).
|
||||
- Tests that built fixtures with `type='l4d2center_maps'` or `type='cedapug_maps'` — those tests likely belong to the deleted set or need conversion to `type='script'`.
|
||||
- Template snapshot tests (if any) that captured the deleted global-source block.
|
||||
|
||||
**Verification:** all three suites green.
|
||||
|
||||
**Commit:** `chore(l4d2-web): test suite drift fixes after script-overlays migration` (only if drift fixes needed; skip if Tasks 1–8 left the suite green)
|
||||
|
||||
---
|
||||
|
||||
## End-to-end deployment verification (manual, on test host)
|
||||
|
||||
After all tasks committed:
|
||||
|
||||
1. **Reset deploy:** run `deploy/deploy-test-server.sh` from clean state. Confirm `bubblewrap` installed (`dpkg -l bubblewrap`), `l4d2-sandbox` user exists (`id l4d2-sandbox`), `/usr/local/libexec/left4me/left4me-script-sandbox` is mode 0755 and root-owned, `sudo -ln` as `left4me` shows the new rule.
|
||||
2. **Sandbox smoke:** as `left4me`, write `/tmp/echo.sh` containing `echo $(whoami) > /overlay/sentinel`. `mkdir -p /var/lib/left4me/overlays/1`. `sudo /usr/local/libexec/left4me/left4me-script-sandbox 1 /tmp/echo.sh`. Confirm `/var/lib/left4me/overlays/1/sentinel` contains `l4d2-sandbox` and is owned by `l4d2-sandbox`. Confirm `/etc/passwd`, `/var/lib/left4me/l4d2web.db`, and `/home` are not visible inside the sandbox by running probe scripts.
|
||||
3. **Resource limits:**
|
||||
- `dd if=/dev/zero of=/overlay/big bs=1M count=25000` → succeeds inside sandbox; `ScriptBuilder._enforce_disk_budget` flags the build failed; `last_build_status='failed'`.
|
||||
- `sleep 7200` → killed at 1 h by `RuntimeMaxSec=3600`.
|
||||
- Memory hog (`python3 -c "x=' '*(5*1024**3)"`) → OOM at 4 GB.
|
||||
4. **App-level happy path:** as a non-admin user, create a script overlay via the UI, paste an old `competitive_rework`-style script, Save → build runs, succeeds, addons appear in `overlays/{id}/left4dead2/`. Stack onto a server blueprint, start the server, verify content mounts via the L4D2 admin console (`map workshop/...`).
|
||||
5. **Wipe:** click Wipe → dir empty (find -delete output in log). Click Rebuild → repopulates. `last_build_status` cycles: `''` → `'ok'`.
|
||||
6. **Scheduler:** start a server using the script overlay; in another browser tab attempt to Rebuild → 409 / scheduler-blocked. Stop server; rebuild succeeds.
|
||||
7. **Audit log:** `journalctl --since "5 min ago" | grep run-` shows transient scopes per build with cgroup memory accounting visible.
|
||||
|
||||
These are not required for any single commit but should pass before declaring the work done.
|
||||
|
|
@ -1,323 +0,0 @@
|
|||
# L4D2 Script Overlays Design
|
||||
|
||||
**Goal:** Add a single new overlay type, `script`, that lets users author arbitrary build recipes as bash and runs them inside a `bubblewrap` + `systemd-run --scope` sandbox. The new type subsumes the existing `l4d2center_maps` and `cedapug_maps` managed-globals overlay types, both of which are removed in the same change. After this work the overlay type list is exactly `workshop` (unchanged) and `script` (new).
|
||||
|
||||
**Approval status:** User-approved design direction. Implementation proceeds in lockstep with the companion plan at `docs/superpowers/plans/2026-05-08-l4d2-script-overlays.md`.
|
||||
|
||||
## Context
|
||||
|
||||
`left4me` users today have two ways to add content to a server: workshop overlays (rich UI for Steam Workshop items via `WorkshopBuilder`) and a pair of managed global-map overlay types (`l4d2center_maps`, `cedapug_maps`) with bespoke parsers, per-item DB rows, ETag-based change detection, and a daily refresh timer. They cannot author arbitrary build recipes.
|
||||
|
||||
The user's previous setup at `ckn-bw/bundles/left4dead2/files/scripts/overlays/` expressed every recipe as a small bash file: `competitive_rework` (GitHub tarball download), `tickrate` (inline `server.cfg` + addon DLL fetch), `standard` (workshop items + admin-list write), `workshop_maps` (workshop collection import), `l4d2center_maps` (CSV-driven map sync). All five fit naturally into a single "run a sandboxed bash script that populates the overlay dir" model.
|
||||
|
||||
The two managed global-map types in the current codebase are over-engineered for what they do — each is essentially "fetch a manifest, download archives, extract VPKs, place in `addons/`." Folding them into the new `script` type eliminates three database tables, two source-parser modules, the `GlobalMapOverlayBuilder`, the `py7zr` dependency, the global-overlay cache root, and the managed-singleton machinery, while letting an admin paste the equivalent shell code (which the user already wrote years ago) into a normal admin-owned, system-wide script overlay.
|
||||
|
||||
The trust model for the sandbox is "semi-public deployment, registered users." The threat surface is one user reading another user's overlay, the application DB, or arbitrary host secrets, plus runaway scripts exhausting disk/CPU/RAM. Network access is *not* restricted — scripts must be able to download from arbitrary URLs (GitHub, l4d2center, Steam CDN). Sandbox boundaries are namespace-based (mount, PID, IPC, UTS, cgroup), not command-allowlist-based; binary-allowlist sandboxing of bash is theatre because of `eval` and `exec`.
|
||||
|
||||
The test deploy DB is wiped as part of rollout; no data migration is performed. Existing user blueprints that reference `l4d2center_maps` or `cedapug_maps` overlay rows do not survive the change in the test environment.
|
||||
|
||||
A scheduled-refresh feature (the daily timer that today drives the global-map types) is intentionally **out of scope for this iteration**. The two existing systemd units and the `flask refresh-global-overlays` CLI command are deleted with no replacement. Refresh is reintroduced in a later iteration designed against concrete needs.
|
||||
|
||||
## Locked Decisions
|
||||
|
||||
1. **Single new overlay type: `script`.** Replaces both managed-globals types. Final type list: `workshop` + `script`. No `tarball`/`inline`/`manual` types — all of those collapse into `script` (with UI templates as a future ergonomics improvement).
|
||||
2. **`Overlay.script` is a DB `TEXT` column** holding the raw bash. No file storage, no revision history in v1. Empty string for `workshop` rows.
|
||||
3. **Build idempotency contract: script runs against the existing overlay dir.** No automatic wipe between builds. Users write `test -f … || curl …`-style guards if they want bandwidth efficiency. A manual "Wipe overlay" button on the detail page resets the dir to empty.
|
||||
4. **No left4me-aware helpers in the sandbox.** The script sees pure bash plus whatever's in `/usr` (RO bind-mount of the host). Workshop items are not exposed via a helper — users wanting workshop content create a `workshop`-type overlay, which has its own first-class UX (thumbnails, collection paste, dedup cache, refresh).
|
||||
5. **Sandbox engine: `bubblewrap` (`bwrap`) inside `systemd-run --scope --collect`.** `systemd-run` provides cgroup v2 limits + walltime kill via `RuntimeMaxSec`; `bwrap` provides the namespace isolation. Both are stable, well-audited, in-tree on Debian.
|
||||
6. **Resource limits (system-wide, not per-overlay):** 1 hour walltime (`RuntimeMaxSec=3600`), 4 GB RAM (`MemoryMax=4G`, `MemorySwapMax=0`), 512 tasks, 200% CPU quota, post-build 20 GB disk cap on `du -sb` of the overlay dir.
|
||||
7. **Network: host-shared.** No `--unshare-net`. Scripts have full outbound. Egress filtering is not in v1; the sandbox prevents reading internal state but does not prevent talking to internal IPs. Acceptable for the current trust model.
|
||||
8. **No auto-seeding of "default" overlays.** Admin manually creates the equivalents of the old `l4d2center-maps`/`cedapug-maps` post-deploy by pasting the bash. The deploy script does not insert overlay rows.
|
||||
9. **Daily/scheduled refresh: out of scope for this iteration.** No `auto_refresh` flag, no timer, no CLI command. Manual rebuild via the detail-page button is the only build trigger after this change.
|
||||
10. **Permissions mirror workshop overlays.** Any logged-in user can create a private (`user_id = me`) script overlay. Admin can create system-wide (`user_id = NULL`). Owner or admin can edit/delete.
|
||||
11. **Failure semantics via `Overlay.last_build_status`** (`'' | 'ok' | 'failed'`). Drives a "rebuild required" badge on the list and detail pages. Server initialization does **not** auto-block on `failed` (matches workshop's current behavior).
|
||||
12. **Wipe is just another sandbox invocation.** The wipe endpoint runs the literal script `find /overlay -mindepth 1 -delete` through the same `left4me-script-sandbox` helper. No second helper, no privilege/UID puzzle (files are owned by `l4d2-sandbox`, who runs the wipe). After a successful wipe, `last_build_status` is reset to `''`. Wipe does **not** auto-enqueue a rebuild — the user decides.
|
||||
13. **Privileged helper: `/usr/local/libexec/left4me/left4me-script-sandbox`.** Same pattern as the existing `left4me-overlay`, `left4me-systemctl`, `left4me-journalctl` helpers. Bash, owned root, mode 0755. The web user invokes it via `sudo -n` per a sudoers fragment. Root is needed to set up the namespaces; bwrap drops to the unprivileged `l4d2-sandbox` UID immediately.
|
||||
14. **Dedicated sandbox UID `l4d2-sandbox`** (system user, `/usr/sbin/nologin`, no home). Owns nothing on the host outside what bwrap binds in. UID-drop happens inside the bwrap invocation via `--uid`/`--gid`.
|
||||
15. **Strict argument validation in the helper.** Overlay id matches `^[0-9]+$`; overlay dir must exist under `/var/lib/left4me/overlays/`; script path must exist. Defense in depth — the real authorization check lives in the web app.
|
||||
16. **Streaming I/O via the existing `run_with_streamed_output` helper.** Same plumbing `WorkshopBuilder` already uses for `steamcmd`/`curl` invocations. No new SSE/log path.
|
||||
|
||||
## Architecture
|
||||
|
||||
```text
|
||||
Overlay row (type=script, script=TEXT, last_build_status)
|
||||
│
|
||||
▼ build_overlay(overlay_id) job
|
||||
│
|
||||
▼ BUILDERS["script"].build(overlay, on_stdout, on_stderr, should_cancel)
|
||||
│
|
||||
▼ ScriptBuilder writes overlay.script → tmpfile, then:
|
||||
│ sudo -n /usr/local/libexec/left4me/left4me-script-sandbox <id> <tmpfile>
|
||||
│
|
||||
▼ Helper validates args, then exec()s:
|
||||
│ systemd-run --scope --collect
|
||||
│ -p MemoryMax=4G -p MemorySwapMax=0
|
||||
│ -p TasksMax=512 -p CPUQuota=200%
|
||||
│ -p RuntimeMaxSec=3600
|
||||
│ -- bwrap [namespace flags...] /bin/bash /script.sh
|
||||
│
|
||||
▼ Inside the sandbox the script sees:
|
||||
│ /overlay ← /var/lib/left4me/overlays/{id} RW (the build target)
|
||||
│ /tmp,/run ← fresh tmpfs RW (ephemeral)
|
||||
│ /usr,/lib,/lib64,/etc/{ssl,resolv.conf,nsswitch} RO (host-curated)
|
||||
│ /proc,/dev ← fresh
|
||||
│ network ← shared with host
|
||||
│ UID/GID ← l4d2-sandbox (no_new_privs implicit in bwrap)
|
||||
│
|
||||
▼ stdout/stderr → run_with_streamed_output → existing job-log SSE stream
|
||||
▼ After exit:
|
||||
│ exit 0 ∧ du -sb /overlay ≤ 20 GB → last_build_status='ok'
|
||||
│ any other outcome → last_build_status='failed'
|
||||
```
|
||||
|
||||
The host library (`l4d2host`) is unchanged. The `KernelOverlayFSMounter` already mounts whatever's at `overlays/{id}/` regardless of how it got there. The Job model and worker model are essentially unchanged — `script` is just another overlay type for the same `build_overlay` operation that today supports `workshop`.
|
||||
|
||||
```python
|
||||
BUILDERS = {
|
||||
"workshop": WorkshopBuilder(),
|
||||
"script": ScriptBuilder(),
|
||||
}
|
||||
```
|
||||
|
||||
## Data Model
|
||||
|
||||
### `Overlay` (modified)
|
||||
|
||||
```text
|
||||
id INTEGER PK AUTOINCREMENT
|
||||
name VARCHAR(255) NOT NULL
|
||||
path VARCHAR(255) NOT NULL -- str(id) for new rows
|
||||
type VARCHAR(16) NOT NULL -- 'workshop' | 'script'
|
||||
user_id INTEGER NULL REFERENCES users(id) -- NULL = system-wide
|
||||
|
||||
script TEXT NOT NULL DEFAULT '' -- new; meaningful for type='script'
|
||||
last_build_status VARCHAR(16) NOT NULL DEFAULT '' -- new; '' | 'ok' | 'failed'
|
||||
|
||||
created_at, updated_at
|
||||
|
||||
UNIQUE INDEX on (name) WHERE user_id IS NULL
|
||||
UNIQUE INDEX on (name, user_id) WHERE user_id IS NOT NULL
|
||||
INDEX on (type, user_id)
|
||||
```
|
||||
|
||||
### Tables removed
|
||||
|
||||
- `global_overlay_item_files`
|
||||
- `global_overlay_items`
|
||||
- `global_overlay_sources`
|
||||
|
||||
Drop order matters for the SQLite migration: drop `_item_files` first (FK to `_items`), then `_items` (FK to `_sources`), then `_sources` (FK to `overlays`).
|
||||
|
||||
### Unchanged
|
||||
|
||||
`WorkshopItem`, `overlay_workshop_items`, `Job` (including `Job.overlay_id` and nullable `Job.user_id`), `Server`, `Blueprint`, etc.
|
||||
|
||||
## Filesystem Layout
|
||||
|
||||
```text
|
||||
${LEFT4ME_ROOT}/
|
||||
overlays/
|
||||
{overlay_id}/ # script writes here; mounted by host
|
||||
left4dead2/... # whatever the script produces
|
||||
workshop_cache/{steam_id}.vpk # workshop type only — unchanged
|
||||
|
||||
# removed:
|
||||
# global_overlay_cache/ # was used by managed-globals types
|
||||
```
|
||||
|
||||
Single tree per overlay. No per-overlay scratch cache (the chosen idempotency model is "script runs against existing dir," so any caching the user wants lives inside the overlay dir and is preserved between builds).
|
||||
|
||||
The sandbox bind-mounts `${LEFT4ME_ROOT}/overlays/{id}/` to `/overlay` (RW). Nothing else under `${LEFT4ME_ROOT}` is visible inside the sandbox.
|
||||
|
||||
## Sandbox
|
||||
|
||||
### Helper script
|
||||
|
||||
`deploy/files/usr/local/libexec/left4me/left4me-script-sandbox`, mode 0755, owned root:
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# args: <overlay_id> <script_path>
|
||||
set -euo pipefail
|
||||
[[ $# -eq 2 ]] || { echo "usage: $0 <overlay_id> <script>" >&2; exit 64; }
|
||||
OVERLAY_ID=$1; SCRIPT=$2
|
||||
[[ "$OVERLAY_ID" =~ ^[0-9]+$ ]] || { echo "bad overlay id" >&2; exit 64; }
|
||||
OVERLAY_DIR=/var/lib/left4me/overlays/$OVERLAY_ID
|
||||
[[ -d $OVERLAY_DIR ]] || { echo "no overlay dir" >&2; exit 65; }
|
||||
[[ -f $SCRIPT ]] || { echo "no script" >&2; exit 65; }
|
||||
|
||||
SBX_UID=$(id -u l4d2-sandbox); SBX_GID=$(id -g l4d2-sandbox)
|
||||
|
||||
exec systemd-run --quiet --scope --collect \
|
||||
-p MemoryMax=4G -p MemorySwapMax=0 -p TasksMax=512 \
|
||||
-p CPUQuota=200% -p RuntimeMaxSec=3600 \
|
||||
-- bwrap \
|
||||
--die-with-parent --new-session \
|
||||
--unshare-pid --unshare-ipc --unshare-uts --unshare-cgroup \
|
||||
--uid "$SBX_UID" --gid "$SBX_GID" \
|
||||
--proc /proc --dev /dev --tmpfs /tmp --tmpfs /run \
|
||||
--ro-bind /usr /usr --ro-bind /lib /lib --ro-bind /lib64 /lib64 \
|
||||
--symlink usr/bin /bin --symlink usr/sbin /sbin \
|
||||
--ro-bind /etc/resolv.conf /etc/resolv.conf \
|
||||
--ro-bind /etc/ssl /etc/ssl \
|
||||
--ro-bind /etc/ca-certificates /etc/ca-certificates \
|
||||
--ro-bind /etc/nsswitch.conf /etc/nsswitch.conf \
|
||||
--bind "$OVERLAY_DIR" /overlay \
|
||||
--chdir /overlay \
|
||||
--setenv HOME /tmp --setenv PATH /usr/bin:/usr/sbin \
|
||||
--setenv OVERLAY /overlay \
|
||||
--ro-bind "$SCRIPT" /script.sh \
|
||||
/bin/bash /script.sh
|
||||
```
|
||||
|
||||
Network is *not* unshared (no `--unshare-net`); the sandbox shares the host network namespace. Every transient unit is visible via `systemctl list-units --type=scope` while running and journaled afterward (`journalctl --user-unit=run-…scope` or system journal depending on invocation).
|
||||
|
||||
### Sudoers fragment
|
||||
|
||||
Append to `deploy/files/etc/sudoers.d/left4me`:
|
||||
|
||||
```
|
||||
left4me ALL=(root) NOPASSWD: /usr/local/libexec/left4me/left4me-script-sandbox
|
||||
```
|
||||
|
||||
### System user
|
||||
|
||||
Provisioned in `deploy/deploy-test-server.sh`:
|
||||
|
||||
```bash
|
||||
useradd --system --no-create-home --shell /usr/sbin/nologin l4d2-sandbox
|
||||
apt-get install -y bubblewrap
|
||||
```
|
||||
|
||||
## Build Lifecycle
|
||||
|
||||
`ScriptBuilder` lives in `l4d2web/services/overlay_builders.py` next to `WorkshopBuilder`:
|
||||
|
||||
```python
|
||||
class ScriptBuilder:
|
||||
def build(self, overlay, *, on_stdout, on_stderr, should_cancel):
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".sh", delete=False) as f:
|
||||
f.write(overlay.script or "")
|
||||
script_path = f.name
|
||||
try:
|
||||
cmd = [
|
||||
"sudo", "-n",
|
||||
"/usr/local/libexec/left4me/left4me-script-sandbox",
|
||||
str(overlay.id), script_path,
|
||||
]
|
||||
run_with_streamed_output(cmd, on_stdout, on_stderr, should_cancel)
|
||||
self._enforce_disk_budget(overlay.id, on_stderr)
|
||||
finally:
|
||||
os.unlink(script_path)
|
||||
|
||||
def _enforce_disk_budget(self, overlay_id, on_stderr):
|
||||
size = subprocess.check_output(["du", "-sb", overlay_path(overlay_id)])
|
||||
if int(size.split()[0]) > 20 * 1024**3:
|
||||
on_stderr("overlay exceeded 20 GB disk cap")
|
||||
raise BuildError("disk-cap-exceeded")
|
||||
```
|
||||
|
||||
`run_with_streamed_output` is the existing helper used by `WorkshopBuilder` for `steamcmd`/`curl` invocations. The `should_cancel` callback fires `kill -TERM` on the sudo-`systemd-run` process tree; cgroup-collect tears down the whole scope on exit.
|
||||
|
||||
The job worker's existing job-completion path writes `Overlay.last_build_status = 'ok'` on success and `'failed'` on any non-zero exit / `BuildError` / cancel. This is a single column update inside the existing transaction; no new infrastructure.
|
||||
|
||||
## UI
|
||||
|
||||
### Create modal (`templates/overlays.html`)
|
||||
|
||||
The existing modal grows one option in the type radio: `Workshop | Script`. Name field unchanged. After insert, the web app generates `path = str(overlay_id)` for new rows (existing pattern).
|
||||
|
||||
### Detail page when `type='script'` (`templates/overlay_detail.html`)
|
||||
|
||||
- Plain styled `<textarea>` for `overlay.script` with a Save button → `POST /overlays/{id}/script`. No CodeMirror dependency in v1 (out of scope; keep frontend dep-light).
|
||||
- "Rebuild" button → `POST /overlays/{id}/build`. Existing pattern from workshop overlays.
|
||||
- "Wipe overlay" button (red, confirm-modal) → `POST /overlays/{id}/wipe`.
|
||||
- `last_build_status` indicator badge: empty / "ok" / "failed".
|
||||
- Live build log via existing SSE plumbing on the related Job row.
|
||||
|
||||
### Detail page when `type='workshop'`: unchanged.
|
||||
|
||||
### Sections removed
|
||||
|
||||
The global-source detail block (`overlay_detail.html` lines 34–46) is deleted along with the managed-globals subsystem.
|
||||
|
||||
## Routes
|
||||
|
||||
`l4d2web/routes/overlay_routes.py` adds:
|
||||
|
||||
| Method | Path | Purpose |
|
||||
|---|---|---|
|
||||
| POST | `/overlays/{id}/script` | Update `script` text. Auto-enqueue coalesced `build_overlay` job. |
|
||||
| POST | `/overlays/{id}/wipe` | Invoke `left4me-script-sandbox` with the literal script `find /overlay -mindepth 1 -delete`. Owner/admin only. Refuses if a `build_overlay` for this overlay is running. After success, set `last_build_status=''`. Does not auto-enqueue a rebuild. |
|
||||
| POST | `/overlays/{id}/build` | Manual rebuild — same pattern as today's workshop overlay manual rebuild. |
|
||||
|
||||
Existing `POST /overlays` accepts `type=script` and an optional initial `script` body.
|
||||
|
||||
## Permissions
|
||||
|
||||
| Action | Who |
|
||||
|---|---|
|
||||
| Create script overlay (private, `user_id = me`) | Any authenticated user |
|
||||
| Create script overlay (system-wide, `user_id = NULL`) | Admin |
|
||||
| Edit (script body, name) | Owner or admin |
|
||||
| Wipe / Rebuild | Owner or admin |
|
||||
| Delete | Owner or admin |
|
||||
| View | Owner, admin, or any user when `user_id IS NULL` |
|
||||
|
||||
These match the existing rules for workshop overlays.
|
||||
|
||||
## Job Worker / Scheduler
|
||||
|
||||
`services/job_worker.py` drops `"refresh_global_overlays"` from `GLOBAL_OPERATIONS` and removes the corresponding `refresh_global_overlays_running` and `blocked_servers_by_overlay` plumbing that exists only for the global-maps subsystem. The remaining mutex rules already cover:
|
||||
|
||||
- `build_overlay` per overlay (one running build per overlay).
|
||||
- `install` and `refresh_workshop_items` as global mutexes.
|
||||
- Server start/init blocks if any `build_overlay` for an overlay in the server's blueprint is running.
|
||||
|
||||
No new rules are needed for `script` — its build is mechanically identical to a `workshop` build from the scheduler's perspective.
|
||||
|
||||
## Daily Refresh — Removed
|
||||
|
||||
This iteration deletes the daily-refresh subsystem entirely:
|
||||
|
||||
- `deploy/files/usr/local/lib/systemd/system/left4me-refresh-global-overlays.timer` and `.service` — deleted.
|
||||
- `flask refresh-global-overlays` CLI command in `l4d2web/cli.py` — deleted.
|
||||
- No replacement timer, no replacement CLI, no `auto_refresh` column on `Overlay`.
|
||||
|
||||
The only build trigger after this change is the user clicking Rebuild on the detail page (or the auto-enqueue when they Save the script body). A scheduled-refresh feature is reintroduced in a future iteration designed against concrete operational needs.
|
||||
|
||||
## Risks
|
||||
|
||||
- **Sandbox escape via kernel bug.** `bwrap` has a strong track record but is not invulnerable. Mitigated by running as `l4d2-sandbox` (no privileged capabilities), no setuid binaries reachable, `no_new_privs` implicit. A successful escape would land in an unprivileged UID with no host secrets reachable.
|
||||
- **Disk fill via runaway script.** A script that writes a 20 GB+ payload to `/overlay` succeeds inside the sandbox and only fails afterward at the post-build `du` check. The 20 GB lands on disk transiently. Mitigated by the kernel's per-cgroup IO accounting being unaware of file size (no good IO-time limit), accepting this as a v1 trade-off; a future improvement is overlay-dir-on-its-own-filesystem with a quota.
|
||||
- **Network exfiltration.** Script can connect to anything outbound, including internal IPs. Acceptable for the current trust model (semi-public; users have credentials). Egress firewall is out of scope.
|
||||
- **Build-mid-server-running.** The scheduler refuses `build_overlay` for an overlay attached to a starting/running server (existing rule, unchanged). Good. A user can still rebuild while a server using a *different* blueprint runs concurrently.
|
||||
- **Wipe race with running build.** The wipe endpoint refuses if a `build_overlay` for the overlay is running. Without this check, a wipe could blow away files mid-script and produce undefined results.
|
||||
- **Stale `last_build_status`.** A row inserted via direct DB write or restored from backup could carry an `'ok'` status that no longer reflects reality. Treated as cosmetic; users can rebuild to refresh.
|
||||
- **Sudoers misconfig.** A typo in the sudoers fragment could grant `left4me` more than intended. Mitigated by deploy-artifact tests asserting the exact expected lines.
|
||||
- **DB row deletion racing the sandbox.** A user deleting an overlay while its build runs would invalidate the bind-mount target. Mitigated by the existing scheduler rule that tracks running overlays; delete should refuse if a build is running. (Existing pattern for workshop overlays; reuse.)
|
||||
- **Migration drops globals tables.** Acceptable for the test deploy. Production rollout would need a different migration story; this spec explicitly assumes test-deploy DB wipe.
|
||||
|
||||
## Out Of Scope
|
||||
|
||||
- **Scheduled / daily refresh.** Intentionally removed in this iteration. Reintroduced later, designed against the use cases that emerge.
|
||||
- **Per-overlay resource overrides.** All script overlays share the same 1 h / 4 GB / 20 GB envelope. If a real overlay needs more (l4d2center mirror at peak), revisit.
|
||||
- **CodeMirror or other rich script editor.** Plain `<textarea>` in v1.
|
||||
- **Egress allowlist / proxy.** No network restrictions on the sandbox in v1.
|
||||
- **`$CACHE` scratch dir** persisted across builds. Users cache inside the overlay dir if they want; idempotency model is "script runs against existing dir."
|
||||
- **Multi-tenant cgroup tree per user.** All sandboxes share the same cgroup-quota envelope.
|
||||
- **Revision history on `script` column.** No `overlay_script_revisions` table; whatever's in the row is the current script.
|
||||
- **Auto-seeding of l4d2center / cedapug equivalents.** Admin pastes the script post-deploy.
|
||||
- **Migration that preserves existing global-map overlay rows.** Test deploy DB is wiped.
|
||||
- **Container-per-build (podman / docker).** Heavier than `bwrap`; revisit only if multi-tenant escalates to "fully public sign-up."
|
||||
- **left4me-aware helpers** (`workshop`, `download`, `extract`) inside the sandbox. Pure bash + host `/usr` only.
|
||||
|
||||
## Implementation Boundaries
|
||||
|
||||
- **`l4d2host` is unchanged.** The host library has no concept of overlay types and the mount layer (`KernelOverlayFSMounter`) doesn't care how the overlay dir got populated.
|
||||
- **The `OverlayBuilder` Protocol is unchanged** — same `build(overlay, *, on_stdout, on_stderr, should_cancel)` signature. `ScriptBuilder` plugs into the existing registry.
|
||||
- **The job worker model is unchanged.** Same operations, same logs, same SSE plumbing, same scheduler rules (minus the refresh_global_overlays entry).
|
||||
- **No new application-level dependencies.** Vendored HTMX, no new Python packages. Two new system dependencies: `bubblewrap` apt package and the `l4d2-sandbox` system user.
|
||||
- **No new config keys.** Same env files (`/etc/left4me/host.env`, `/etc/left4me/web.env`).
|
||||
- **DB migration is destructive for global-maps overlay rows.** This is acceptable per the test-deploy assumption; a production-rollout follow-up would need to address it.
|
||||
- The companion implementation plan governs task ordering and verification commands. Implementation must not start without explicit user approval per that plan's gate.
|
||||
|
|
@ -1,79 +0,0 @@
|
|||
"""script overlays
|
||||
|
||||
Revision ID: 0005_script_overlays
|
||||
Revises: 0004_drop_legacy_external_overlay_type
|
||||
Create Date: 2026-05-08
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision: str = "0005_script_overlays"
|
||||
down_revision: Union[str, Sequence[str], None] = "0004_drop_legacy_external_overlay_type"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# 1. Wipe legacy global-type overlay rows and any references to them.
|
||||
op.execute(
|
||||
"DELETE FROM jobs "
|
||||
"WHERE overlay_id IN (SELECT id FROM overlays "
|
||||
"WHERE type IN ('l4d2center_maps', 'cedapug_maps'))"
|
||||
)
|
||||
op.execute(
|
||||
"DELETE FROM blueprint_overlays "
|
||||
"WHERE overlay_id IN (SELECT id FROM overlays "
|
||||
"WHERE type IN ('l4d2center_maps', 'cedapug_maps'))"
|
||||
)
|
||||
op.execute(
|
||||
"DELETE FROM overlay_workshop_items "
|
||||
"WHERE overlay_id IN (SELECT id FROM overlays "
|
||||
"WHERE type IN ('l4d2center_maps', 'cedapug_maps'))"
|
||||
)
|
||||
op.execute(
|
||||
"DELETE FROM overlays WHERE type IN ('l4d2center_maps', 'cedapug_maps')"
|
||||
)
|
||||
|
||||
# 2. Drop globals tables in FK order: item_files -> items -> sources.
|
||||
op.drop_index(
|
||||
"ix_global_overlay_item_files_item",
|
||||
table_name="global_overlay_item_files",
|
||||
)
|
||||
op.drop_table("global_overlay_item_files")
|
||||
|
||||
op.drop_index(
|
||||
"ix_global_overlay_items_source", table_name="global_overlay_items"
|
||||
)
|
||||
op.drop_table("global_overlay_items")
|
||||
|
||||
op.drop_index(
|
||||
"ix_global_overlay_sources_type", table_name="global_overlay_sources"
|
||||
)
|
||||
op.drop_table("global_overlay_sources")
|
||||
|
||||
# 3. Add new columns on overlays.
|
||||
with op.batch_alter_table("overlays") as batch_op:
|
||||
batch_op.add_column(
|
||||
sa.Column(
|
||||
"script",
|
||||
sa.Text(),
|
||||
nullable=False,
|
||||
server_default="",
|
||||
)
|
||||
)
|
||||
batch_op.add_column(
|
||||
sa.Column(
|
||||
"last_build_status",
|
||||
sa.String(length=16),
|
||||
nullable=False,
|
||||
server_default="",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# data is gone; intentional one-way migration
|
||||
pass
|
||||
|
|
@ -41,6 +41,20 @@ def create_user(username: str, admin: bool) -> None:
|
|||
click.echo(f"created user {username}")
|
||||
|
||||
|
||||
@click.command("refresh-global-overlays")
|
||||
def refresh_global_overlays_command() -> None:
|
||||
from l4d2web.services.global_overlays import (
|
||||
ensure_global_overlays,
|
||||
enqueue_refresh_global_overlays,
|
||||
)
|
||||
|
||||
with session_scope() as db:
|
||||
ensure_global_overlays(db)
|
||||
job = enqueue_refresh_global_overlays(db, user_id=None)
|
||||
click.echo(f"queued refresh_global_overlays job #{job.id}")
|
||||
|
||||
|
||||
def register_cli(app) -> None:
|
||||
app.cli.add_command(promote_admin)
|
||||
app.cli.add_command(create_user)
|
||||
app.cli.add_command(refresh_global_overlays_command)
|
||||
|
|
|
|||
|
|
@ -59,8 +59,69 @@ class Overlay(Base):
|
|||
path: Mapped[str] = mapped_column(String(512), nullable=False)
|
||||
type: Mapped[str] = mapped_column(String(16), nullable=False, default="workshop")
|
||||
user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id"), nullable=True)
|
||||
script: Mapped[str] = mapped_column(Text, default="", nullable=False)
|
||||
last_build_status: Mapped[str] = mapped_column(String(16), default="", nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, default=now_utc, nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime, default=now_utc, nullable=False)
|
||||
|
||||
|
||||
class GlobalOverlaySource(Base):
|
||||
__tablename__ = "global_overlay_sources"
|
||||
__table_args__ = (Index("ix_global_overlay_sources_type", "source_type"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
overlay_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("overlays.id", ondelete="CASCADE"), unique=True, nullable=False
|
||||
)
|
||||
source_key: Mapped[str] = mapped_column(String(64), unique=True, nullable=False)
|
||||
source_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
source_url: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
last_manifest_hash: Mapped[str] = mapped_column(String(64), default="", nullable=False)
|
||||
last_refreshed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
||||
last_error: Mapped[str] = mapped_column(Text, default="", nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, default=now_utc, nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime, default=now_utc, nullable=False)
|
||||
|
||||
|
||||
class GlobalOverlayItem(Base):
|
||||
__tablename__ = "global_overlay_items"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("source_id", "item_key", name="uq_global_overlay_item_source_key"),
|
||||
Index("ix_global_overlay_items_source", "source_id"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
source_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("global_overlay_sources.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
item_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
display_name: Mapped[str] = mapped_column(String(255), default="", nullable=False)
|
||||
download_url: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
expected_vpk_name: Mapped[str] = mapped_column(String(255), default="", nullable=False)
|
||||
expected_size: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
||||
expected_md5: Mapped[str] = mapped_column(String(32), default="", nullable=False)
|
||||
etag: Mapped[str] = mapped_column(String(255), default="", nullable=False)
|
||||
last_modified: Mapped[str] = mapped_column(String(255), default="", nullable=False)
|
||||
content_length: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
||||
last_downloaded_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
||||
last_error: Mapped[str] = mapped_column(Text, default="", nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, default=now_utc, nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime, default=now_utc, nullable=False)
|
||||
|
||||
|
||||
class GlobalOverlayItemFile(Base):
|
||||
__tablename__ = "global_overlay_item_files"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("item_id", "vpk_name", name="uq_global_overlay_item_file_name"),
|
||||
Index("ix_global_overlay_item_files_item", "item_id"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
item_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("global_overlay_items.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
vpk_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
cache_path: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
size: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||
md5: Mapped[str] = mapped_column(String(32), default="", nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, default=now_utc, nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime, default=now_utc, nullable=False)
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ dependencies = [
|
|||
"PyYAML>=6.0",
|
||||
"gunicorn>=22.0",
|
||||
"requests>=2.31",
|
||||
"py7zr>=0.21",
|
||||
]
|
||||
|
||||
[tool.setuptools]
|
||||
|
|
|
|||
|
|
@ -7,19 +7,14 @@ from l4d2host.paths import get_left4me_root
|
|||
|
||||
from l4d2web.auth import current_user, require_login
|
||||
from l4d2web.db import session_scope
|
||||
from l4d2web.models import BlueprintOverlay, Job, Overlay
|
||||
from l4d2web.services import overlay_builders
|
||||
from l4d2web.services.job_worker import enqueue_build_overlay
|
||||
from l4d2web.models import BlueprintOverlay, Overlay
|
||||
from l4d2web.services.global_overlays import MANAGED_GLOBAL_OVERLAY_TYPES, is_creatable_overlay_type
|
||||
from l4d2web.services.overlay_creation import (
|
||||
create_overlay_directory,
|
||||
generate_overlay_path,
|
||||
)
|
||||
|
||||
|
||||
CREATABLE_OVERLAY_TYPES = {"workshop", "script"}
|
||||
WIPE_SCRIPT = "find /overlay -mindepth 1 -delete"
|
||||
|
||||
|
||||
bp = Blueprint("overlay", __name__)
|
||||
|
||||
|
||||
|
|
@ -30,9 +25,11 @@ def _is_managed_path(overlay: Overlay) -> bool:
|
|||
def _can_edit_overlay(overlay: Overlay, user) -> bool:
|
||||
if user is None:
|
||||
return False
|
||||
if overlay.type in MANAGED_GLOBAL_OVERLAY_TYPES:
|
||||
return False
|
||||
if user.admin:
|
||||
return True
|
||||
if overlay.type in {"workshop", "script"}:
|
||||
if overlay.type == "workshop":
|
||||
return overlay.user_id == user.id
|
||||
return False
|
||||
|
||||
|
|
@ -56,13 +53,12 @@ def create_overlay() -> Response:
|
|||
|
||||
name = request.form.get("name", "").strip()
|
||||
overlay_type = request.form.get("type", "workshop").strip().lower()
|
||||
system_wide = request.form.get("system_wide") == "1"
|
||||
if not name:
|
||||
return Response("missing fields", status=400)
|
||||
if overlay_type not in CREATABLE_OVERLAY_TYPES:
|
||||
if not is_creatable_overlay_type(overlay_type, admin=user.admin):
|
||||
return Response(f"unknown overlay type: {overlay_type}", status=400)
|
||||
|
||||
scope_user_id: int | None = None if (system_wide and user.admin) else user.id
|
||||
scope_user_id: int | None = user.id
|
||||
|
||||
with session_scope() as db:
|
||||
if _name_already_taken(db, name, scope_user_id):
|
||||
|
|
@ -127,80 +123,3 @@ def delete_overlay(overlay_id: int) -> Response:
|
|||
shutil.rmtree(target)
|
||||
|
||||
return redirect("/overlays")
|
||||
|
||||
|
||||
def _load_script_overlay(db, overlay_id: int, user) -> tuple[Overlay | None, Response | None]:
|
||||
overlay = db.scalar(select(Overlay).where(Overlay.id == overlay_id))
|
||||
if overlay is None:
|
||||
return None, Response(status=404)
|
||||
if overlay.type != "script":
|
||||
return None, Response("not a script overlay", status=400)
|
||||
if not _can_edit_overlay(overlay, user):
|
||||
return None, Response(status=403)
|
||||
return overlay, None
|
||||
|
||||
|
||||
@bp.post("/overlays/<int:overlay_id>/script")
|
||||
@require_login
|
||||
def update_script(overlay_id: int) -> Response:
|
||||
user = current_user()
|
||||
assert user is not None
|
||||
script_text = request.form.get("script", "")
|
||||
with session_scope() as db:
|
||||
overlay, err = _load_script_overlay(db, overlay_id, user)
|
||||
if err is not None:
|
||||
return err
|
||||
overlay.script = script_text
|
||||
enqueue_build_overlay(db, overlay_id=overlay_id, user_id=user.id)
|
||||
return redirect(f"/overlays/{overlay_id}")
|
||||
|
||||
|
||||
@bp.post("/overlays/<int:overlay_id>/build")
|
||||
@require_login
|
||||
def manual_build(overlay_id: int) -> Response:
|
||||
user = current_user()
|
||||
assert user is not None
|
||||
with session_scope() as db:
|
||||
overlay = db.scalar(select(Overlay).where(Overlay.id == overlay_id))
|
||||
if overlay is None:
|
||||
return Response(status=404)
|
||||
if not _can_edit_overlay(overlay, user):
|
||||
return Response(status=403)
|
||||
job = enqueue_build_overlay(db, overlay_id=overlay_id, user_id=user.id)
|
||||
job_id = job.id
|
||||
return redirect(f"/jobs/{job_id}")
|
||||
|
||||
|
||||
@bp.post("/overlays/<int:overlay_id>/wipe")
|
||||
@require_login
|
||||
def wipe_overlay(overlay_id: int) -> Response:
|
||||
user = current_user()
|
||||
assert user is not None
|
||||
with session_scope() as db:
|
||||
overlay, err = _load_script_overlay(db, overlay_id, user)
|
||||
if err is not None:
|
||||
return err
|
||||
running = db.scalar(
|
||||
select(Job).where(
|
||||
Job.operation == "build_overlay",
|
||||
Job.overlay_id == overlay_id,
|
||||
Job.state.in_({"running", "cancelling"}),
|
||||
)
|
||||
)
|
||||
if running is not None:
|
||||
return Response("build in progress for this overlay", status=409)
|
||||
|
||||
overlay_builders.run_sandboxed_script(
|
||||
overlay_id,
|
||||
WIPE_SCRIPT,
|
||||
on_stdout=lambda _line: None,
|
||||
on_stderr=lambda _line: None,
|
||||
should_cancel=lambda: False,
|
||||
)
|
||||
|
||||
with session_scope() as db:
|
||||
overlay = db.scalar(select(Overlay).where(Overlay.id == overlay_id))
|
||||
if overlay is not None:
|
||||
overlay.last_build_status = ""
|
||||
|
||||
return redirect(f"/overlays/{overlay_id}")
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from l4d2web.db import session_scope
|
|||
from l4d2web.models import Blueprint as BlueprintModel
|
||||
from l4d2web.models import (
|
||||
BlueprintOverlay,
|
||||
GlobalOverlaySource,
|
||||
Job,
|
||||
Overlay,
|
||||
OverlayWorkshopItem,
|
||||
|
|
@ -42,6 +43,22 @@ def enqueue_runtime_install() -> Response:
|
|||
return redirect("/admin/jobs")
|
||||
|
||||
|
||||
@bp.post("/admin/global-overlays/refresh")
|
||||
@require_admin
|
||||
def enqueue_global_overlay_refresh() -> Response:
|
||||
user = current_user()
|
||||
assert user is not None
|
||||
from l4d2web.services.global_overlays import (
|
||||
ensure_global_overlays,
|
||||
enqueue_refresh_global_overlays,
|
||||
)
|
||||
|
||||
with session_scope() as db:
|
||||
ensure_global_overlays(db)
|
||||
enqueue_refresh_global_overlays(db, user_id=user.id)
|
||||
return redirect("/admin/jobs")
|
||||
|
||||
|
||||
@bp.get("/admin/users")
|
||||
@require_admin
|
||||
def admin_users() -> str:
|
||||
|
|
@ -172,6 +189,9 @@ def overlay_detail(overlay_id: int):
|
|||
return Response(status=404)
|
||||
if not user.admin and overlay.user_id is not None and overlay.user_id != user.id:
|
||||
return Response(status=403)
|
||||
global_source = db.scalar(
|
||||
select(GlobalOverlaySource).where(GlobalOverlaySource.overlay_id == overlay.id)
|
||||
)
|
||||
using_blueprints_query = (
|
||||
select(BlueprintModel)
|
||||
.join(BlueprintOverlay, BlueprintOverlay.blueprint_id == BlueprintModel.id)
|
||||
|
|
@ -201,6 +221,7 @@ def overlay_detail(overlay_id: int):
|
|||
return render_template(
|
||||
"overlay_detail.html",
|
||||
overlay=overlay,
|
||||
global_source=global_source,
|
||||
using_blueprints=using_blueprints,
|
||||
workshop_items=workshop_items,
|
||||
latest_build_job=latest_build_job,
|
||||
|
|
|
|||
|
|
@ -142,6 +142,20 @@ def remove_item(overlay_id: int, item_id: int) -> Response:
|
|||
return _render_item_table(overlay_id)
|
||||
|
||||
|
||||
@bp.post("/overlays/<int:overlay_id>/build")
|
||||
@require_login
|
||||
def manual_build(overlay_id: int) -> Response:
|
||||
user = current_user()
|
||||
assert user is not None
|
||||
with session_scope() as db:
|
||||
overlay, err = _check_workshop_overlay_access(overlay_id, user, db)
|
||||
if err is not None:
|
||||
return err
|
||||
job = enqueue_build_overlay(db, overlay_id=overlay_id, user_id=user.id)
|
||||
job_id = job.id
|
||||
return redirect(f"/jobs/{job_id}")
|
||||
|
||||
|
||||
@bp.post("/admin/workshop/refresh")
|
||||
@require_admin
|
||||
def admin_refresh() -> Response:
|
||||
|
|
|
|||
106
l4d2web/services/global_map_cache.py
Normal file
106
l4d2web/services/global_map_cache.py
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
from zipfile import ZipFile
|
||||
|
||||
import py7zr
|
||||
import requests
|
||||
|
||||
from l4d2host.paths import get_left4me_root
|
||||
|
||||
|
||||
REQUEST_TIMEOUT_SECONDS = 30
|
||||
DOWNLOAD_CHUNK_BYTES = 1_048_576
|
||||
|
||||
|
||||
def global_overlay_cache_root() -> Path:
|
||||
return get_left4me_root() / "global_overlay_cache"
|
||||
|
||||
|
||||
def source_cache_root(source_key: str) -> Path:
|
||||
if "/" in source_key or ".." in source_key or not source_key:
|
||||
raise ValueError(f"invalid source_key: {source_key!r}")
|
||||
return global_overlay_cache_root() / source_key
|
||||
|
||||
|
||||
def archive_dir(source_key: str) -> Path:
|
||||
return source_cache_root(source_key) / "archives"
|
||||
|
||||
|
||||
def vpk_dir(source_key: str) -> Path:
|
||||
return source_cache_root(source_key) / "vpks"
|
||||
|
||||
|
||||
def download_archive(url: str, target: Path, *, should_cancel=None) -> tuple[str, str, int | None]:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
partial = target.with_suffix(target.suffix + ".partial")
|
||||
response = requests.get(url, stream=True, timeout=REQUEST_TIMEOUT_SECONDS)
|
||||
response.raise_for_status()
|
||||
etag = response.headers.get("ETag", "")
|
||||
last_modified = response.headers.get("Last-Modified", "")
|
||||
content_length_raw = response.headers.get("Content-Length")
|
||||
content_length = int(content_length_raw) if content_length_raw and content_length_raw.isdigit() else None
|
||||
try:
|
||||
with open(partial, "wb") as f:
|
||||
for chunk in response.iter_content(chunk_size=DOWNLOAD_CHUNK_BYTES):
|
||||
if should_cancel is not None and should_cancel():
|
||||
raise InterruptedError("download cancelled")
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
os.replace(partial, target)
|
||||
except BaseException:
|
||||
partial.unlink(missing_ok=True)
|
||||
raise
|
||||
return etag, last_modified, content_length
|
||||
|
||||
|
||||
def safe_extract_zip_vpks(archive_path: Path, output_dir: Path) -> list[Path]:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
extracted: list[Path] = []
|
||||
with ZipFile(archive_path) as zf:
|
||||
for member in zf.infolist():
|
||||
name = Path(member.filename)
|
||||
if name.is_absolute() or any(part in {"", ".", ".."} for part in name.parts):
|
||||
raise ValueError(f"unsafe archive member: {member.filename}")
|
||||
if name.suffix.lower() != ".vpk":
|
||||
continue
|
||||
target = output_dir / name.name
|
||||
with zf.open(member) as src, open(target, "wb") as dst:
|
||||
shutil.copyfileobj(src, dst)
|
||||
extracted.append(target)
|
||||
if not extracted:
|
||||
raise ValueError(f"archive {archive_path} did not contain any .vpk files")
|
||||
return sorted(extracted)
|
||||
|
||||
|
||||
def safe_extract_7z_vpks(archive_path: Path, output_dir: Path) -> list[Path]:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix="left4me-7z-") as raw_tmp:
|
||||
raw_dir = Path(raw_tmp)
|
||||
with py7zr.SevenZipFile(archive_path, mode="r") as archive:
|
||||
names = archive.getnames()
|
||||
for name in names:
|
||||
p = Path(name)
|
||||
if p.is_absolute() or any(part in {"", ".", ".."} for part in p.parts):
|
||||
raise ValueError(f"unsafe archive member: {name}")
|
||||
archive.extractall(path=raw_dir)
|
||||
extracted: list[Path] = []
|
||||
for candidate in raw_dir.rglob("*.vpk"):
|
||||
target = output_dir / candidate.name
|
||||
shutil.move(str(candidate), str(target))
|
||||
extracted.append(target)
|
||||
if not extracted:
|
||||
raise ValueError(f"archive {archive_path} did not contain any .vpk files")
|
||||
return sorted(extracted)
|
||||
|
||||
|
||||
def extracted_vpk_md5(path: Path) -> str:
|
||||
digest = hashlib.md5()
|
||||
with open(path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
104
l4d2web/services/global_map_sources.py
Normal file
104
l4d2web/services/global_map_sources.py
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
import html as html_lib
|
||||
import io
|
||||
import json
|
||||
from urllib.parse import urljoin, urlparse
|
||||
import re
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
REQUEST_TIMEOUT_SECONDS = 30
|
||||
L4D2CENTER_CSV_URL = "https://l4d2center.com/maps/servers/index.csv"
|
||||
CEDAPUG_CUSTOM_URL = "https://cedapug.com/custom"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GlobalMapManifestItem:
|
||||
item_key: str
|
||||
display_name: str
|
||||
download_url: str
|
||||
expected_vpk_name: str = ""
|
||||
expected_size: int | None = None
|
||||
expected_md5: str = ""
|
||||
|
||||
|
||||
def fetch_l4d2center_manifest() -> tuple[str, list[GlobalMapManifestItem]]:
|
||||
response = requests.get(L4D2CENTER_CSV_URL, timeout=REQUEST_TIMEOUT_SECONDS)
|
||||
response.raise_for_status()
|
||||
text = response.text
|
||||
return _sha256(text), parse_l4d2center_csv(text)
|
||||
|
||||
|
||||
def fetch_cedapug_manifest() -> tuple[str, list[GlobalMapManifestItem]]:
|
||||
response = requests.get(CEDAPUG_CUSTOM_URL, timeout=REQUEST_TIMEOUT_SECONDS)
|
||||
response.raise_for_status()
|
||||
text = response.text
|
||||
return _sha256(text), parse_cedapug_custom_html(text)
|
||||
|
||||
|
||||
def parse_l4d2center_csv(raw: str) -> list[GlobalMapManifestItem]:
|
||||
reader = csv.DictReader(io.StringIO(raw), delimiter=";")
|
||||
expected = ["Name", "Size", "md5", "Download link"]
|
||||
if reader.fieldnames != expected:
|
||||
raise ValueError("expected L4D2Center CSV header: Name;Size;md5;Download link")
|
||||
items: list[GlobalMapManifestItem] = []
|
||||
for row in reader:
|
||||
name = (row.get("Name") or "").strip()
|
||||
size_raw = (row.get("Size") or "").strip()
|
||||
md5 = (row.get("md5") or "").strip().lower()
|
||||
url = (row.get("Download link") or "").strip()
|
||||
if not name or not url:
|
||||
continue
|
||||
items.append(
|
||||
GlobalMapManifestItem(
|
||||
item_key=name,
|
||||
display_name=name,
|
||||
download_url=url,
|
||||
expected_vpk_name=name,
|
||||
expected_size=int(size_raw) if size_raw else None,
|
||||
expected_md5=md5,
|
||||
)
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
def parse_cedapug_custom_html(raw: str) -> list[GlobalMapManifestItem]:
|
||||
match = re.search(r"renderCustomMapDownloads\((\[.*?\])\)</script>", raw, re.DOTALL)
|
||||
if match is None:
|
||||
raise ValueError("CEDAPUG page did not contain renderCustomMapDownloads data")
|
||||
rows = json.loads(match.group(1))
|
||||
items: list[GlobalMapManifestItem] = []
|
||||
for row in rows:
|
||||
if len(row) < 3:
|
||||
continue
|
||||
label = str(row[1])
|
||||
link = str(row[2])
|
||||
if link.startswith("http"):
|
||||
continue
|
||||
if not link:
|
||||
continue
|
||||
url = urljoin(CEDAPUG_CUSTOM_URL, link)
|
||||
parsed = urlparse(url)
|
||||
basename = parsed.path.rsplit("/", 1)[-1]
|
||||
items.append(
|
||||
GlobalMapManifestItem(
|
||||
item_key=basename,
|
||||
display_name=_strip_html(label),
|
||||
download_url=url,
|
||||
)
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
def _strip_html(raw: str) -> str:
|
||||
no_tags = re.sub(r"<[^>]+>", "", raw)
|
||||
return html_lib.unescape(no_tags).strip()
|
||||
|
||||
|
||||
def _sha256(raw: str) -> str:
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||
168
l4d2web/services/global_overlay_refresh.py
Normal file
168
l4d2web/services/global_overlay_refresh.py
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from l4d2web.db import session_scope
|
||||
from l4d2web.models import GlobalOverlayItem, GlobalOverlayItemFile, GlobalOverlaySource, Overlay
|
||||
from l4d2web.services.global_map_cache import (
|
||||
archive_dir,
|
||||
download_archive,
|
||||
extracted_vpk_md5,
|
||||
safe_extract_7z_vpks,
|
||||
safe_extract_zip_vpks,
|
||||
vpk_dir,
|
||||
)
|
||||
from l4d2web.services.global_map_sources import (
|
||||
GlobalMapManifestItem,
|
||||
fetch_cedapug_manifest,
|
||||
fetch_l4d2center_manifest,
|
||||
)
|
||||
from l4d2web.services.global_overlays import ensure_global_overlays
|
||||
|
||||
|
||||
def refresh_global_overlays(*, on_stdout, on_stderr, should_cancel) -> list[str]:
|
||||
with session_scope() as db:
|
||||
ensure_global_overlays(db)
|
||||
|
||||
refreshed: list[str] = []
|
||||
for source_key, fetcher in (
|
||||
("l4d2center-maps", fetch_l4d2center_manifest),
|
||||
("cedapug-maps", fetch_cedapug_manifest),
|
||||
):
|
||||
if should_cancel():
|
||||
on_stderr("global overlay refresh cancelled before manifest fetch")
|
||||
return refreshed
|
||||
manifest_hash, manifest_items = fetcher()
|
||||
on_stdout(f"{source_key}: fetched manifest with {len(manifest_items)} item(s)")
|
||||
overlay = _refresh_source(
|
||||
source_key,
|
||||
manifest_hash,
|
||||
manifest_items,
|
||||
on_stdout=on_stdout,
|
||||
on_stderr=on_stderr,
|
||||
should_cancel=should_cancel,
|
||||
)
|
||||
build_global_overlay(overlay, on_stdout=on_stdout, on_stderr=on_stderr, should_cancel=should_cancel)
|
||||
refreshed.append(source_key)
|
||||
return sorted(refreshed)
|
||||
|
||||
|
||||
def _refresh_source(source_key: str, manifest_hash: str, manifest_items: list[GlobalMapManifestItem], *, on_stdout, on_stderr, should_cancel) -> Overlay:
|
||||
now = datetime.now(UTC)
|
||||
desired_keys = {item.item_key for item in manifest_items}
|
||||
with session_scope() as db:
|
||||
source = db.scalar(select(GlobalOverlaySource).where(GlobalOverlaySource.source_key == source_key))
|
||||
if source is None:
|
||||
raise ValueError(f"global overlay source {source_key!r} not found")
|
||||
overlay = db.scalar(select(Overlay).where(Overlay.id == source.overlay_id))
|
||||
if overlay is None:
|
||||
raise ValueError(f"overlay for source {source_key!r} not found")
|
||||
existing_items = {item.item_key: item for item in db.scalars(select(GlobalOverlayItem).where(GlobalOverlayItem.source_id == source.id)).all()}
|
||||
for old_key, old_item in list(existing_items.items()):
|
||||
if old_key not in desired_keys:
|
||||
db.delete(old_item)
|
||||
for manifest_item in manifest_items:
|
||||
item = existing_items.get(manifest_item.item_key)
|
||||
if item is None:
|
||||
item = GlobalOverlayItem(source_id=source.id, item_key=manifest_item.item_key, download_url=manifest_item.download_url)
|
||||
db.add(item)
|
||||
db.flush()
|
||||
item.display_name = manifest_item.display_name
|
||||
item.download_url = manifest_item.download_url
|
||||
item.expected_vpk_name = manifest_item.expected_vpk_name
|
||||
item.expected_size = manifest_item.expected_size
|
||||
item.expected_md5 = manifest_item.expected_md5
|
||||
item.updated_at = now
|
||||
source.last_manifest_hash = manifest_hash
|
||||
source.last_refreshed_at = now
|
||||
source.last_error = ""
|
||||
source.updated_at = now
|
||||
db.expunge(overlay)
|
||||
|
||||
for manifest_item in manifest_items:
|
||||
if should_cancel():
|
||||
on_stderr(f"{source_key}: refresh cancelled during downloads")
|
||||
return overlay
|
||||
_refresh_item(source_key, manifest_item, on_stdout=on_stdout, on_stderr=on_stderr, should_cancel=should_cancel)
|
||||
return overlay
|
||||
|
||||
|
||||
def _refresh_item(source_key: str, manifest_item: GlobalMapManifestItem, *, on_stdout, on_stderr, should_cancel) -> None:
|
||||
try:
|
||||
files, etag, last_modified, content_length = download_and_extract_item(source_key, manifest_item, should_cancel=should_cancel)
|
||||
except Exception as exc:
|
||||
with session_scope() as db:
|
||||
source = db.scalar(select(GlobalOverlaySource).where(GlobalOverlaySource.source_key == source_key))
|
||||
if source is not None:
|
||||
item = db.scalar(select(GlobalOverlayItem).where(GlobalOverlayItem.source_id == source.id, GlobalOverlayItem.item_key == manifest_item.item_key))
|
||||
if item is not None:
|
||||
item.last_error = str(exc)
|
||||
on_stderr(f"{source_key}: {manifest_item.item_key}: {exc}")
|
||||
return
|
||||
|
||||
now = datetime.now(UTC)
|
||||
with session_scope() as db:
|
||||
source = db.scalar(select(GlobalOverlaySource).where(GlobalOverlaySource.source_key == source_key))
|
||||
if source is None:
|
||||
raise ValueError(f"global overlay source {source_key!r} not found")
|
||||
item = db.scalar(select(GlobalOverlayItem).where(GlobalOverlayItem.source_id == source.id, GlobalOverlayItem.item_key == manifest_item.item_key))
|
||||
if item is None:
|
||||
raise ValueError(f"global overlay item {manifest_item.item_key!r} not found")
|
||||
db.query(GlobalOverlayItemFile).filter_by(item_id=item.id).delete()
|
||||
for vpk_name, cache_path, size, md5 in files:
|
||||
db.add(GlobalOverlayItemFile(item_id=item.id, vpk_name=vpk_name, cache_path=cache_path, size=size, md5=md5))
|
||||
item.etag = etag
|
||||
item.last_modified = last_modified
|
||||
item.content_length = content_length
|
||||
item.last_downloaded_at = now
|
||||
item.last_error = ""
|
||||
item.updated_at = now
|
||||
on_stdout(f"{source_key}: refreshed {manifest_item.item_key} ({len(files)} vpk file(s))")
|
||||
|
||||
|
||||
def download_and_extract_item(source_key: str, item: GlobalMapManifestItem, *, should_cancel) -> tuple[list[tuple[str, str, int, str]], str, str, int | None]:
|
||||
archives = archive_dir(source_key)
|
||||
vpks = vpk_dir(source_key)
|
||||
archives.mkdir(parents=True, exist_ok=True)
|
||||
vpks.mkdir(parents=True, exist_ok=True)
|
||||
archive_name = item.download_url.rsplit("/", 1)[-1]
|
||||
archive_path = archives / archive_name
|
||||
etag, last_modified, content_length = download_archive(item.download_url, archive_path, should_cancel=should_cancel)
|
||||
with tempfile.TemporaryDirectory(prefix="left4me-global-map-") as tmp:
|
||||
tmp_dir = Path(tmp)
|
||||
if archive_name.lower().endswith(".7z"):
|
||||
extracted = safe_extract_7z_vpks(archive_path, tmp_dir)
|
||||
elif archive_name.lower().endswith(".zip"):
|
||||
extracted = safe_extract_zip_vpks(archive_path, tmp_dir)
|
||||
else:
|
||||
raise ValueError(f"unsupported archive extension for {archive_name}")
|
||||
results: list[tuple[str, str, int, str]] = []
|
||||
for path in extracted:
|
||||
if item.expected_vpk_name and path.name != item.expected_vpk_name:
|
||||
continue
|
||||
size = path.stat().st_size
|
||||
md5 = extracted_vpk_md5(path)
|
||||
if item.expected_size is not None and size != item.expected_size:
|
||||
raise ValueError(f"{path.name} size mismatch: expected {item.expected_size}, got {size}")
|
||||
if item.expected_md5 and md5 != item.expected_md5:
|
||||
raise ValueError(f"{path.name} md5 mismatch: expected {item.expected_md5}, got {md5}")
|
||||
final = vpks / path.name
|
||||
shutil.move(str(path), str(final))
|
||||
results.append((path.name, f"{source_key}/vpks/{path.name}", size, md5))
|
||||
if not results:
|
||||
raise ValueError(f"no expected .vpk files extracted from {archive_name}")
|
||||
return results, etag, last_modified, content_length
|
||||
|
||||
|
||||
def build_global_overlay(overlay: Overlay, *, on_stdout, on_stderr, should_cancel) -> None:
|
||||
from l4d2web.services.overlay_builders import BUILDERS
|
||||
|
||||
builder = BUILDERS.get(overlay.type)
|
||||
if builder is None:
|
||||
raise ValueError(f"no builder registered for overlay type {overlay.type!r}")
|
||||
builder.build(overlay, on_stdout=on_stdout, on_stderr=on_stderr, should_cancel=should_cancel)
|
||||
112
l4d2web/services/global_overlays.py
Normal file
112
l4d2web/services/global_overlays.py
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import os
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from l4d2host.paths import get_left4me_root
|
||||
|
||||
from l4d2web.models import GlobalOverlaySource, Job, Overlay
|
||||
from l4d2web.services.overlay_creation import generate_overlay_path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ManagedGlobalOverlay:
|
||||
name: str
|
||||
overlay_type: str
|
||||
source_type: str
|
||||
source_url: str
|
||||
|
||||
|
||||
GLOBAL_OVERLAYS = (
|
||||
ManagedGlobalOverlay(
|
||||
name="l4d2center-maps",
|
||||
overlay_type="l4d2center_maps",
|
||||
source_type="l4d2center_csv",
|
||||
source_url="https://l4d2center.com/maps/servers/index.csv",
|
||||
),
|
||||
ManagedGlobalOverlay(
|
||||
name="cedapug-maps",
|
||||
overlay_type="cedapug_maps",
|
||||
source_type="cedapug_custom_page",
|
||||
source_url="https://cedapug.com/custom",
|
||||
),
|
||||
)
|
||||
|
||||
MANAGED_GLOBAL_OVERLAY_TYPES = {overlay.overlay_type for overlay in GLOBAL_OVERLAYS}
|
||||
USER_CREATABLE_TYPES = {"workshop"}
|
||||
ADMIN_CREATABLE_TYPES = {"workshop"}
|
||||
|
||||
|
||||
def is_creatable_overlay_type(overlay_type: str, *, admin: bool) -> bool:
|
||||
allowed = ADMIN_CREATABLE_TYPES if admin else USER_CREATABLE_TYPES
|
||||
return overlay_type in allowed
|
||||
|
||||
|
||||
def ensure_global_overlays(session: Session) -> set[str]:
|
||||
created_sources: set[str] = set()
|
||||
for managed in GLOBAL_OVERLAYS:
|
||||
overlay = session.scalar(
|
||||
select(Overlay).where(Overlay.name == managed.name, Overlay.user_id.is_(None))
|
||||
)
|
||||
overlay_created = overlay is None
|
||||
if overlay is None:
|
||||
overlay = Overlay(name=managed.name, path="", type=managed.overlay_type, user_id=None)
|
||||
session.add(overlay)
|
||||
session.flush()
|
||||
overlay.path = generate_overlay_path(overlay.id)
|
||||
else:
|
||||
overlay.type = managed.overlay_type
|
||||
overlay.user_id = None
|
||||
if not overlay.path:
|
||||
overlay.path = generate_overlay_path(overlay.id)
|
||||
|
||||
target = get_left4me_root() / "overlays" / overlay.path
|
||||
os.makedirs(target, exist_ok=not overlay_created)
|
||||
|
||||
source = session.scalar(
|
||||
select(GlobalOverlaySource).where(GlobalOverlaySource.source_key == managed.name)
|
||||
)
|
||||
if source is None:
|
||||
source = GlobalOverlaySource(
|
||||
overlay_id=overlay.id,
|
||||
source_key=managed.name,
|
||||
source_type=managed.source_type,
|
||||
source_url=managed.source_url,
|
||||
)
|
||||
session.add(source)
|
||||
created_sources.add(managed.name)
|
||||
else:
|
||||
source.overlay_id = overlay.id
|
||||
source.source_type = managed.source_type
|
||||
source.source_url = managed.source_url
|
||||
|
||||
session.flush()
|
||||
|
||||
return created_sources
|
||||
|
||||
|
||||
def enqueue_refresh_global_overlays(session: Session, *, user_id: int | None) -> Job:
|
||||
existing = session.scalar(
|
||||
select(Job)
|
||||
.where(
|
||||
Job.operation == "refresh_global_overlays",
|
||||
Job.state.in_({"queued", "running", "cancelling"}),
|
||||
)
|
||||
.order_by(Job.created_at, Job.id)
|
||||
)
|
||||
if existing is not None:
|
||||
return existing
|
||||
|
||||
job = Job(
|
||||
user_id=user_id,
|
||||
server_id=None,
|
||||
overlay_id=None,
|
||||
operation="refresh_global_overlays",
|
||||
state="queued",
|
||||
)
|
||||
session.add(job)
|
||||
session.flush()
|
||||
return job
|
||||
|
|
@ -27,7 +27,7 @@ TERMINAL_JOB_STATES = {"succeeded", "failed", "cancelled"}
|
|||
ACTIVE_JOB_STATES = {"running", "cancelling"}
|
||||
SERVER_OPERATIONS = {"initialize", "start", "stop", "delete"}
|
||||
OVERLAY_OPERATIONS = {"build_overlay"}
|
||||
GLOBAL_OPERATIONS = {"install", "refresh_workshop_items"}
|
||||
GLOBAL_OPERATIONS = {"install", "refresh_workshop_items", "refresh_global_overlays"}
|
||||
WORKSHOP_REFRESH_DOWNLOAD_WORKERS = 1
|
||||
|
||||
_claim_lock = threading.Lock()
|
||||
|
|
@ -40,6 +40,7 @@ _workers_started = False
|
|||
class SchedulerState:
|
||||
install_running: bool = False
|
||||
refresh_running: bool = False
|
||||
refresh_global_overlays_running: bool = False
|
||||
running_servers: set[int] = field(default_factory=set)
|
||||
running_overlays: set[int] = field(default_factory=set)
|
||||
blocked_servers_by_overlay: set[int] = field(default_factory=set)
|
||||
|
|
@ -62,6 +63,7 @@ def can_start(job, state: SchedulerState) -> bool:
|
|||
return (
|
||||
not state.install_running
|
||||
and not state.refresh_running
|
||||
and not state.refresh_global_overlays_running
|
||||
and len(state.running_servers) == 0
|
||||
and len(state.running_overlays) == 0
|
||||
)
|
||||
|
|
@ -69,17 +71,26 @@ def can_start(job, state: SchedulerState) -> bool:
|
|||
return (
|
||||
not state.install_running
|
||||
and not state.refresh_running
|
||||
and not state.refresh_global_overlays_running
|
||||
and len(state.running_servers) == 0
|
||||
and len(state.running_overlays) == 0
|
||||
)
|
||||
if job.operation == "refresh_global_overlays":
|
||||
return (
|
||||
not state.install_running
|
||||
and not state.refresh_running
|
||||
and not state.refresh_global_overlays_running
|
||||
and len(state.running_servers) == 0
|
||||
and len(state.running_overlays) == 0
|
||||
)
|
||||
if job.operation == "build_overlay":
|
||||
if state.install_running or state.refresh_running:
|
||||
if state.install_running or state.refresh_running or state.refresh_global_overlays_running:
|
||||
return False
|
||||
if job.overlay_id is None:
|
||||
return False
|
||||
return job.overlay_id not in state.running_overlays
|
||||
# Server operations from here on.
|
||||
if state.install_running or state.refresh_running:
|
||||
if state.install_running or state.refresh_running or state.refresh_global_overlays_running:
|
||||
return False
|
||||
if job.server_id is None:
|
||||
return False
|
||||
|
|
@ -98,6 +109,8 @@ def build_scheduler_state(session: Session) -> SchedulerState:
|
|||
state.install_running = True
|
||||
elif job.operation == "refresh_workshop_items":
|
||||
state.refresh_running = True
|
||||
elif job.operation == "refresh_global_overlays":
|
||||
state.refresh_global_overlays_running = True
|
||||
elif job.operation == "build_overlay" and job.overlay_id is not None:
|
||||
state.running_overlays.add(job.overlay_id)
|
||||
elif job.server_id is not None:
|
||||
|
|
@ -247,6 +260,15 @@ def run_job(job_id: int) -> None:
|
|||
on_stderr=on_stderr,
|
||||
should_cancel=should_cancel,
|
||||
)
|
||||
elif operation == "refresh_global_overlays":
|
||||
_run_with_boundaries(
|
||||
"refresh",
|
||||
"global overlays",
|
||||
_run_refresh_global_overlays,
|
||||
on_stdout=on_stdout,
|
||||
on_stderr=on_stderr,
|
||||
should_cancel=should_cancel,
|
||||
)
|
||||
elif operation == "build_overlay":
|
||||
if overlay_id_for_job is None:
|
||||
raise ValueError("build_overlay job has no overlay_id")
|
||||
|
|
@ -368,6 +390,21 @@ def _run_build_overlay(
|
|||
)
|
||||
|
||||
|
||||
def _run_refresh_global_overlays(
|
||||
*,
|
||||
on_stdout: Callable[[str], None],
|
||||
on_stderr: Callable[[str], None],
|
||||
should_cancel: Callable[[], bool],
|
||||
) -> list[str]:
|
||||
from l4d2web.services.global_overlay_refresh import refresh_global_overlays
|
||||
|
||||
return refresh_global_overlays(
|
||||
on_stdout=on_stdout,
|
||||
on_stderr=on_stderr,
|
||||
should_cancel=should_cancel,
|
||||
)
|
||||
|
||||
|
||||
def _run_refresh_workshop_items(
|
||||
*,
|
||||
on_stdout: Callable[[str], None],
|
||||
|
|
@ -503,11 +540,6 @@ def finish_job(job_id: int, state: str, exit_code: int | None, error: str = "")
|
|||
if server is not None:
|
||||
server.last_error = "" if state == "succeeded" else error
|
||||
server.updated_at = now
|
||||
if job.operation == "build_overlay" and job.overlay_id is not None:
|
||||
overlay = db.scalar(select(Overlay).where(Overlay.id == job.overlay_id))
|
||||
if overlay is not None:
|
||||
overlay.last_build_status = "ok" if state == "succeeded" else "failed"
|
||||
overlay.updated_at = now
|
||||
|
||||
|
||||
def append_job_log_line(job_id: int, stream: str, line: str, max_chars: int = 4096) -> int:
|
||||
|
|
|
|||
|
|
@ -8,12 +8,16 @@ from l4d2web.db import session_scope
|
|||
from l4d2web.models import (
|
||||
Blueprint,
|
||||
BlueprintOverlay,
|
||||
GlobalOverlayItem,
|
||||
GlobalOverlayItemFile,
|
||||
GlobalOverlaySource,
|
||||
Overlay,
|
||||
OverlayWorkshopItem,
|
||||
Server,
|
||||
WorkshopItem,
|
||||
)
|
||||
from l4d2web.services import host_commands
|
||||
from l4d2web.services.global_map_cache import global_overlay_cache_root
|
||||
from l4d2web.services.spec_yaml import write_temp_spec
|
||||
from l4d2web.services.workshop_paths import cache_path
|
||||
|
||||
|
|
@ -79,6 +83,7 @@ def initialize_server(server_id: int, on_stdout=None, on_stderr=None, should_can
|
|||
# them, but we don't want to mount a partial overlay silently — fail
|
||||
# loudly with the missing IDs.
|
||||
_check_workshop_overlay_caches(blueprint_id=blueprint.id)
|
||||
_check_global_overlay_caches(blueprint_id=blueprint.id)
|
||||
|
||||
spec_path = write_temp_spec(build_server_spec_payload(server, blueprint, overlay_refs))
|
||||
try:
|
||||
|
|
@ -173,6 +178,36 @@ def _check_workshop_overlay_caches(*, blueprint_id: int) -> None:
|
|||
)
|
||||
|
||||
|
||||
def _check_global_overlay_caches(*, blueprint_id: int) -> None:
|
||||
"""Raise if any global map overlay attached to this blueprint has manifest
|
||||
items that aren't yet in the global_overlay_cache. Mirrors the workshop
|
||||
cache check — surface partial cache state at initialize time.
|
||||
"""
|
||||
with session_scope() as db:
|
||||
rows = db.execute(
|
||||
select(Overlay.name, GlobalOverlayItemFile.vpk_name, GlobalOverlayItemFile.cache_path)
|
||||
.join(BlueprintOverlay, BlueprintOverlay.overlay_id == Overlay.id)
|
||||
.join(GlobalOverlaySource, GlobalOverlaySource.overlay_id == Overlay.id)
|
||||
.join(GlobalOverlayItem, GlobalOverlayItem.source_id == GlobalOverlaySource.id)
|
||||
.join(GlobalOverlayItemFile, GlobalOverlayItemFile.item_id == GlobalOverlayItem.id)
|
||||
.where(BlueprintOverlay.blueprint_id == blueprint_id)
|
||||
).all()
|
||||
|
||||
missing: dict[str, list[str]] = {}
|
||||
root = global_overlay_cache_root()
|
||||
for overlay_name, vpk_name, cache_path_value in rows:
|
||||
if not (root / cache_path_value).exists():
|
||||
missing.setdefault(overlay_name, []).append(vpk_name)
|
||||
|
||||
if not missing:
|
||||
return
|
||||
|
||||
details = []
|
||||
for overlay_name, names in sorted(missing.items()):
|
||||
details.append(f"overlay {overlay_name!r}: missing {', '.join(sorted(names))}")
|
||||
raise RuntimeError("global overlay content missing — " + "; ".join(details))
|
||||
|
||||
|
||||
def start_server(server_id: int, on_stdout=None, on_stderr=None, should_cancel=None) -> None:
|
||||
server, _, _ = load_server_blueprint_bundle(server_id)
|
||||
host_commands.run_command(
|
||||
|
|
|
|||
|
|
@ -8,8 +8,6 @@ changes to the worker, the mount layer, or the blueprint editor.
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Callable, Protocol
|
||||
|
||||
|
|
@ -18,8 +16,8 @@ from sqlalchemy import select
|
|||
from l4d2host.paths import get_left4me_root
|
||||
|
||||
from l4d2web.db import session_scope
|
||||
from l4d2web.models import Overlay, OverlayWorkshopItem, WorkshopItem
|
||||
from l4d2web.services.host_commands import run_command
|
||||
from l4d2web.models import GlobalOverlayItem, GlobalOverlayItemFile, GlobalOverlaySource, Overlay, OverlayWorkshopItem, WorkshopItem
|
||||
from l4d2web.services.global_map_cache import global_overlay_cache_root
|
||||
from l4d2web.services.workshop_paths import cache_path, workshop_cache_root
|
||||
|
||||
|
||||
|
|
@ -27,16 +25,6 @@ CancelCheck = Callable[[], bool]
|
|||
LogSink = Callable[[str], None]
|
||||
|
||||
|
||||
SCRIPT_SANDBOX_HELPER = "/usr/local/libexec/left4me/left4me-script-sandbox"
|
||||
DISK_BUDGET_BYTES = 20 * 1024**3
|
||||
|
||||
|
||||
class BuildError(RuntimeError):
|
||||
"""Raised by builders when a build fails for a builder-specific reason
|
||||
(e.g. disk-budget exceeded). Distinct from subprocess-level
|
||||
HostCommandError / CommandCancelledError."""
|
||||
|
||||
|
||||
class OverlayBuilder(Protocol):
|
||||
def build(
|
||||
self,
|
||||
|
|
@ -52,10 +40,6 @@ def _overlay_root(overlay: Overlay) -> Path:
|
|||
return get_left4me_root() / "overlays" / overlay.path
|
||||
|
||||
|
||||
def overlay_path_for_id(overlay_id: int) -> Path:
|
||||
return get_left4me_root() / "overlays" / str(overlay_id)
|
||||
|
||||
|
||||
class WorkshopBuilder:
|
||||
"""Diff-apply symlinks under `left4dead2/addons/` against the overlay's
|
||||
current `WorkshopItem` associations. Cached items get an absolute symlink
|
||||
|
|
@ -179,45 +163,8 @@ class WorkshopBuilder:
|
|||
)
|
||||
|
||||
|
||||
def run_sandboxed_script(
|
||||
overlay_id: int,
|
||||
script_text: str,
|
||||
*,
|
||||
on_stdout: LogSink,
|
||||
on_stderr: LogSink,
|
||||
should_cancel: CancelCheck,
|
||||
) -> None:
|
||||
"""Write `script_text` to a tmpfile and exec it inside the privileged
|
||||
sandbox helper. Used by ScriptBuilder.build and by the wipe route."""
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".sh", delete=False) as f:
|
||||
f.write(script_text or "")
|
||||
script_path = f.name
|
||||
try:
|
||||
cmd = [
|
||||
"sudo",
|
||||
"-n",
|
||||
SCRIPT_SANDBOX_HELPER,
|
||||
str(overlay_id),
|
||||
script_path,
|
||||
]
|
||||
run_command(
|
||||
cmd,
|
||||
on_stdout=on_stdout,
|
||||
on_stderr=on_stderr,
|
||||
should_cancel=should_cancel,
|
||||
)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(script_path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
class ScriptBuilder:
|
||||
"""Run an arbitrary user-authored bash script against the overlay dir
|
||||
inside a bubblewrap + systemd-run sandbox. The script sees the overlay
|
||||
dir as RW `/overlay` and a curated host RO mount; everything else is
|
||||
isolated. After exit, enforce a 20 GB cap on `du -sb /overlay`."""
|
||||
class GlobalMapOverlayBuilder:
|
||||
"""Reconcile symlinks for managed global map overlays."""
|
||||
|
||||
def build(
|
||||
self,
|
||||
|
|
@ -227,28 +174,84 @@ class ScriptBuilder:
|
|||
on_stderr: LogSink,
|
||||
should_cancel: CancelCheck,
|
||||
) -> None:
|
||||
# Ensure target dir exists so the helper's bind-mount validation passes.
|
||||
overlay_path_for_id(overlay.id).mkdir(parents=True, exist_ok=True)
|
||||
addons_dir = _overlay_root(overlay) / "left4dead2" / "addons"
|
||||
addons_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
run_sandboxed_script(
|
||||
overlay.id,
|
||||
overlay.script or "",
|
||||
on_stdout=on_stdout,
|
||||
on_stderr=on_stderr,
|
||||
should_cancel=should_cancel,
|
||||
)
|
||||
self._enforce_disk_budget(overlay.id, on_stderr)
|
||||
with session_scope() as db:
|
||||
source = db.scalar(select(GlobalOverlaySource).where(GlobalOverlaySource.overlay_id == overlay.id))
|
||||
if source is None:
|
||||
raise ValueError(f"global overlay source for overlay {overlay.id} not found")
|
||||
rows = db.execute(
|
||||
select(GlobalOverlayItemFile.vpk_name, GlobalOverlayItemFile.cache_path)
|
||||
.join(GlobalOverlayItem, GlobalOverlayItem.id == GlobalOverlayItemFile.item_id)
|
||||
.where(GlobalOverlayItem.source_id == source.id)
|
||||
).all()
|
||||
source_key = source.source_key
|
||||
|
||||
def _enforce_disk_budget(self, overlay_id: int, on_stderr: LogSink) -> None:
|
||||
target = overlay_path_for_id(overlay_id)
|
||||
size_output = subprocess.check_output(["du", "-sb", str(target)])
|
||||
size_bytes = int(size_output.split()[0])
|
||||
if size_bytes > DISK_BUDGET_BYTES:
|
||||
on_stderr(
|
||||
f"overlay exceeded 20 GB disk cap: {size_bytes} bytes > "
|
||||
f"{DISK_BUDGET_BYTES} bytes"
|
||||
cache_root = global_overlay_cache_root().resolve()
|
||||
source_vpk_root = (global_overlay_cache_root() / source_key / "vpks").resolve()
|
||||
desired: dict[str, Path] = {}
|
||||
skipped = 0
|
||||
for vpk_name, cache_path_value in rows:
|
||||
target = (global_overlay_cache_root() / cache_path_value).resolve()
|
||||
if not _is_under(target, source_vpk_root) or not target.exists():
|
||||
on_stderr(f"global overlay {overlay.name!r}: missing cache file for {vpk_name}")
|
||||
skipped += 1
|
||||
continue
|
||||
desired[vpk_name] = target
|
||||
|
||||
existing: dict[str, Path] = {}
|
||||
for entry in os.scandir(addons_dir):
|
||||
if not entry.is_symlink():
|
||||
continue
|
||||
try:
|
||||
resolved = Path(os.readlink(entry.path)).resolve(strict=False)
|
||||
except OSError:
|
||||
continue
|
||||
if _is_under(resolved, source_vpk_root):
|
||||
existing[entry.name] = resolved
|
||||
elif _is_under(resolved, cache_root):
|
||||
on_stderr(f"global overlay {overlay.name!r}: leaving foreign cache symlink {entry.name}")
|
||||
|
||||
created = 0
|
||||
removed = 0
|
||||
unchanged = 0
|
||||
for name, current_target in existing.items():
|
||||
if should_cancel():
|
||||
on_stderr("global overlay build cancelled mid-removal")
|
||||
return
|
||||
desired_target = desired.get(name)
|
||||
if desired_target is None:
|
||||
os.unlink(addons_dir / name)
|
||||
removed += 1
|
||||
elif current_target == desired_target:
|
||||
unchanged += 1
|
||||
else:
|
||||
os.unlink(addons_dir / name)
|
||||
|
||||
current_names = {
|
||||
name for name, current_target in existing.items() if name in desired and current_target == desired[name]
|
||||
}
|
||||
for name, target in desired.items():
|
||||
if should_cancel():
|
||||
on_stderr("global overlay build cancelled mid-creation")
|
||||
return
|
||||
if name in current_names:
|
||||
continue
|
||||
link_path = addons_dir / name
|
||||
if link_path.exists() and not link_path.is_symlink():
|
||||
on_stderr(f"refusing to overwrite non-symlink at {link_path}")
|
||||
continue
|
||||
if link_path.is_symlink():
|
||||
on_stderr(f"refusing to overwrite foreign symlink at {link_path}")
|
||||
continue
|
||||
os.symlink(str(target), str(link_path))
|
||||
created += 1
|
||||
|
||||
on_stdout(
|
||||
f"global overlay {overlay.name!r}: created={created} removed={removed} "
|
||||
f"unchanged={unchanged} skipped(missing)={skipped}"
|
||||
)
|
||||
raise BuildError("disk-cap-exceeded")
|
||||
|
||||
|
||||
def _is_under(path: Path, root: Path) -> bool:
|
||||
|
|
@ -261,5 +264,6 @@ def _is_under(path: Path, root: Path) -> bool:
|
|||
|
||||
BUILDERS: dict[str, OverlayBuilder] = {
|
||||
"workshop": WorkshopBuilder(),
|
||||
"script": ScriptBuilder(),
|
||||
"l4d2center_maps": GlobalMapOverlayBuilder(),
|
||||
"cedapug_maps": GlobalMapOverlayBuilder(),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
<section class="panel">
|
||||
<div class="page-heading">
|
||||
<h1>Overlay: {{ overlay.name }}</h1>
|
||||
{% set can_edit = g.user.admin or (overlay.type in ['workshop', 'script'] and overlay.user_id == g.user.id) %}
|
||||
{% set can_edit = overlay.type not in ['l4d2center_maps', 'cedapug_maps'] and (g.user.admin or (overlay.type == 'workshop' and overlay.user_id == g.user.id)) %}
|
||||
{% if can_edit %}
|
||||
<button type="button" class="danger" data-modal-open="delete-overlay-modal">Delete</button>
|
||||
{% endif %}
|
||||
|
|
@ -27,58 +27,21 @@
|
|||
<tr><th>Type</th><td>{{ overlay.type }}</td></tr>
|
||||
<tr><th>Scope</th><td>{% if overlay.user_id %}private{% else %}system{% endif %}</td></tr>
|
||||
<tr><th>Path</th><td class="muted">{{ overlay.path }}</td></tr>
|
||||
<tr>
|
||||
<th>Last build</th>
|
||||
<td>
|
||||
{% if overlay.last_build_status == 'ok' %}
|
||||
<span class="badge badge-ok">ok</span>
|
||||
{% elif overlay.last_build_status == 'failed' %}
|
||||
<span class="badge badge-error">failed</span>
|
||||
{% else %}
|
||||
<span class="badge badge-muted">never</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
|
||||
{% if overlay.type == 'script' %}
|
||||
{% if global_source %}
|
||||
<section class="panel">
|
||||
<div class="page-heading">
|
||||
<h2>Script</h2>
|
||||
{% if can_edit %}
|
||||
<div class="inline-form-group">
|
||||
<form method="post" action="/overlays/{{ overlay.id }}/build" class="inline-form">
|
||||
<input type="hidden" name="csrf_token" value="{{ session.get('csrf_token', '') }}">
|
||||
<button type="submit" class="button-secondary">Rebuild</button>
|
||||
</form>
|
||||
<button type="button" class="danger" data-modal-open="wipe-overlay-modal">Wipe</button>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
{% if can_edit %}
|
||||
<form method="post" action="/overlays/{{ overlay.id }}/script" class="stack">
|
||||
<input type="hidden" name="csrf_token" value="{{ session.get('csrf_token', '') }}">
|
||||
<label>Bash script
|
||||
<textarea name="script" rows="20" spellcheck="false">{{ overlay.script or "" }}</textarea>
|
||||
</label>
|
||||
<p class="muted">Runs sandboxed against the overlay directory mounted at <code>/overlay</code>. Saving auto-enqueues a build.</p>
|
||||
<div>
|
||||
<button type="submit">Save</button>
|
||||
</div>
|
||||
</form>
|
||||
{% else %}
|
||||
<pre class="script-preview">{{ overlay.script or "" }}</pre>
|
||||
{% endif %}
|
||||
|
||||
{% if latest_build_job %}
|
||||
<p>
|
||||
Latest build: <a href="/jobs/{{ latest_build_job.id }}">job #{{ latest_build_job.id }}</a>
|
||||
— state: <strong>{{ latest_build_job.state }}</strong>
|
||||
</p>
|
||||
{% endif %}
|
||||
<h2>Global source</h2>
|
||||
<table class="definition-table">
|
||||
<tbody>
|
||||
<tr><th>Source key</th><td>{{ global_source.source_key }}</td></tr>
|
||||
<tr><th>Source URL</th><td><a href="{{ global_source.source_url }}">{{ global_source.source_url }}</a></td></tr>
|
||||
<tr><th>Last refreshed</th><td>{{ global_source.last_refreshed_at or "Never" }}</td></tr>
|
||||
<tr><th>Last error</th><td>{{ global_source.last_error or "None" }}</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
|
|
@ -155,24 +118,5 @@
|
|||
</form>
|
||||
</div>
|
||||
</dialog>
|
||||
|
||||
{% if overlay.type == 'script' %}
|
||||
<dialog id="wipe-overlay-modal" class="modal" aria-labelledby="wipe-overlay-title">
|
||||
<div class="modal-header">
|
||||
<h2 id="wipe-overlay-title">Wipe overlay "{{ overlay.name }}"?</h2>
|
||||
<button type="button" class="modal-close" data-modal-close aria-label="Close">×</button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<p>Empties the overlay directory. Use Rebuild afterward to repopulate.</p>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="button-secondary" data-modal-close>Cancel</button>
|
||||
<form method="post" action="/overlays/{{ overlay.id }}/wipe" class="inline-form">
|
||||
<input type="hidden" name="csrf_token" value="{{ session.get('csrf_token', '') }}">
|
||||
<button class="danger" type="submit">Wipe</button>
|
||||
</form>
|
||||
</div>
|
||||
</dialog>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
|
|
|
|||
|
|
@ -37,12 +37,8 @@
|
|||
<fieldset class="overlay-type-radio">
|
||||
<legend>Type</legend>
|
||||
<label><input type="radio" name="type" value="workshop" checked> Workshop (downloads from Steam)</label>
|
||||
<label><input type="radio" name="type" value="script"> Script (runs sandboxed bash)</label>
|
||||
</fieldset>
|
||||
<label>Name <input name="name" required></label>
|
||||
{% if g.user and g.user.admin %}
|
||||
<label><input type="checkbox" name="system_wide" value="1"> System-wide (visible to all users)</label>
|
||||
{% endif %}
|
||||
<p class="muted">The path is generated automatically.</p>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
|
|
|
|||
|
|
@ -1,96 +0,0 @@
|
|||
"""Tests for the alembic migration history.
|
||||
|
||||
The 0005 migration adds `script` and `last_build_status` columns to `overlays`,
|
||||
drops the global_overlay_* tables, and wipes legacy l4d2center_maps/cedapug_maps
|
||||
overlay rows. This module pins those behaviors.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from sqlalchemy import create_engine, inspect, text
|
||||
|
||||
|
||||
_ALEMBIC_DIR = Path(__file__).resolve().parents[1] / "alembic"
|
||||
|
||||
|
||||
def _alembic_config(db_url: str) -> Config:
|
||||
cfg = Config()
|
||||
cfg.set_main_option("script_location", str(_ALEMBIC_DIR))
|
||||
cfg.set_main_option("sqlalchemy.url", db_url)
|
||||
return cfg
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db_url(tmp_path, monkeypatch):
|
||||
path = tmp_path / "alembic.db"
|
||||
url = f"sqlite:///{path}"
|
||||
monkeypatch.setenv("DATABASE_URL", url)
|
||||
yield url
|
||||
|
||||
|
||||
def test_upgrade_0005_adds_script_columns(db_url) -> None:
|
||||
cfg = _alembic_config(db_url)
|
||||
|
||||
command.upgrade(cfg, "0004_drop_legacy_external_overlay_type")
|
||||
|
||||
engine = create_engine(db_url)
|
||||
with engine.begin() as conn:
|
||||
# Seed legacy global-type overlay rows that the migration must wipe.
|
||||
conn.execute(
|
||||
text(
|
||||
"INSERT INTO overlays (name, path, type, created_at, updated_at) "
|
||||
"VALUES ('legacy-l4d2center', '1', 'l4d2center_maps', "
|
||||
"'2026-01-01', '2026-01-01')"
|
||||
)
|
||||
)
|
||||
conn.execute(
|
||||
text(
|
||||
"INSERT INTO overlays (name, path, type, created_at, updated_at) "
|
||||
"VALUES ('legacy-cedapug', '2', 'cedapug_maps', "
|
||||
"'2026-01-01', '2026-01-01')"
|
||||
)
|
||||
)
|
||||
conn.execute(
|
||||
text(
|
||||
"INSERT INTO overlays (name, path, type, created_at, updated_at) "
|
||||
"VALUES ('keep-workshop', '3', 'workshop', "
|
||||
"'2026-01-01', '2026-01-01')"
|
||||
)
|
||||
)
|
||||
|
||||
command.upgrade(cfg, "0005_script_overlays")
|
||||
|
||||
inspector = inspect(engine)
|
||||
|
||||
overlay_cols = {c["name"]: c for c in inspector.get_columns("overlays")}
|
||||
assert "script" in overlay_cols
|
||||
assert "last_build_status" in overlay_cols
|
||||
assert overlay_cols["script"]["nullable"] is False
|
||||
assert overlay_cols["last_build_status"]["nullable"] is False
|
||||
|
||||
table_names = set(inspector.get_table_names())
|
||||
assert "global_overlay_sources" not in table_names
|
||||
assert "global_overlay_items" not in table_names
|
||||
assert "global_overlay_item_files" not in table_names
|
||||
|
||||
with engine.connect() as conn:
|
||||
rows = conn.execute(
|
||||
text("SELECT name, type FROM overlays ORDER BY name")
|
||||
).all()
|
||||
assert rows == [("keep-workshop", "workshop")]
|
||||
|
||||
defaults = conn.execute(
|
||||
text(
|
||||
"SELECT script, last_build_status FROM overlays "
|
||||
"WHERE name = 'keep-workshop'"
|
||||
)
|
||||
).one()
|
||||
assert defaults == ("", "")
|
||||
|
||||
|
||||
def test_downgrade_0005_skipped() -> None:
|
||||
"""Per the project convention (see 0004) destructive migrations are
|
||||
intentionally one-way; do not test or maintain a downgrade."""
|
||||
pytest.skip("0005 is one-way: globals data is gone after upgrade")
|
||||
49
l4d2web/tests/test_global_map_cache.py
Normal file
49
l4d2web/tests/test_global_map_cache.py
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
from pathlib import Path
|
||||
from zipfile import ZipFile
|
||||
|
||||
from l4d2web.services.global_map_cache import (
|
||||
extracted_vpk_md5,
|
||||
global_overlay_cache_root,
|
||||
safe_extract_zip_vpks,
|
||||
source_cache_root,
|
||||
)
|
||||
|
||||
|
||||
def test_global_overlay_cache_paths(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("LEFT4ME_ROOT", str(tmp_path))
|
||||
|
||||
assert global_overlay_cache_root() == tmp_path / "global_overlay_cache"
|
||||
assert source_cache_root("l4d2center-maps") == tmp_path / "global_overlay_cache" / "l4d2center-maps"
|
||||
|
||||
|
||||
def test_safe_extract_zip_vpks_extracts_only_vpks(tmp_path):
|
||||
archive = tmp_path / "maps.zip"
|
||||
with ZipFile(archive, "w") as zf:
|
||||
zf.writestr("FatalFreight.vpk", b"vpk-bytes")
|
||||
zf.writestr("readme.txt", b"ignore")
|
||||
|
||||
out_dir = tmp_path / "out"
|
||||
files = safe_extract_zip_vpks(archive, out_dir)
|
||||
|
||||
assert files == [out_dir / "FatalFreight.vpk"]
|
||||
assert (out_dir / "FatalFreight.vpk").read_bytes() == b"vpk-bytes"
|
||||
assert not (out_dir / "readme.txt").exists()
|
||||
|
||||
|
||||
def test_safe_extract_zip_vpks_rejects_path_traversal(tmp_path):
|
||||
archive = tmp_path / "bad.zip"
|
||||
with ZipFile(archive, "w") as zf:
|
||||
zf.writestr("../evil.vpk", b"bad")
|
||||
|
||||
try:
|
||||
safe_extract_zip_vpks(archive, tmp_path / "out")
|
||||
except ValueError as exc:
|
||||
assert "unsafe archive member" in str(exc)
|
||||
else:
|
||||
raise AssertionError("path traversal must fail")
|
||||
|
||||
|
||||
def test_extracted_vpk_md5(tmp_path):
|
||||
p = tmp_path / "x.vpk"
|
||||
p.write_bytes(b"abc")
|
||||
assert extracted_vpk_md5(p) == "900150983cd24fb0d6963f7d28e17f72"
|
||||
65
l4d2web/tests/test_global_map_sources.py
Normal file
65
l4d2web/tests/test_global_map_sources.py
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
from l4d2web.services.global_map_sources import (
|
||||
GlobalMapManifestItem,
|
||||
parse_cedapug_custom_html,
|
||||
parse_l4d2center_csv,
|
||||
)
|
||||
|
||||
|
||||
def test_parse_l4d2center_csv_semicolon_manifest():
|
||||
raw = """Name;Size;md5;Download link
|
||||
carriedoff.vpk;128660532;0380e12c57156574e17a96da1252cf21;https://l4d2center.com/maps/servers/carriedoff.7z
|
||||
"""
|
||||
|
||||
items = parse_l4d2center_csv(raw)
|
||||
|
||||
assert items == [
|
||||
GlobalMapManifestItem(
|
||||
item_key="carriedoff.vpk",
|
||||
display_name="carriedoff.vpk",
|
||||
download_url="https://l4d2center.com/maps/servers/carriedoff.7z",
|
||||
expected_vpk_name="carriedoff.vpk",
|
||||
expected_size=128660532,
|
||||
expected_md5="0380e12c57156574e17a96da1252cf21",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_parse_l4d2center_rejects_missing_header():
|
||||
try:
|
||||
parse_l4d2center_csv("bad,data\n")
|
||||
except ValueError as exc:
|
||||
assert "Name;Size;md5;Download link" in str(exc)
|
||||
else:
|
||||
raise AssertionError("bad header must fail")
|
||||
|
||||
|
||||
def test_parse_cedapug_custom_html_extracts_relative_zip_links():
|
||||
html = """
|
||||
<script>renderCustomMapDownloads([
|
||||
["c1m1_hotel","<span style='color: #977d4c;'>Dead Center<\\/span>"],
|
||||
["l4d2_ff01_woods","<span style='color: #854C34;'>Fatal Freight<\\/span>","\\/maps\\/FatalFreight.zip"],
|
||||
["external","External","https://steamcommunity.com/sharedfiles/filedetails/?id=123"]
|
||||
])</script>
|
||||
"""
|
||||
|
||||
items = parse_cedapug_custom_html(html)
|
||||
|
||||
assert items == [
|
||||
GlobalMapManifestItem(
|
||||
item_key="FatalFreight.zip",
|
||||
display_name="Fatal Freight",
|
||||
download_url="https://cedapug.com/maps/FatalFreight.zip",
|
||||
expected_vpk_name="",
|
||||
expected_size=None,
|
||||
expected_md5="",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_parse_cedapug_custom_html_rejects_missing_data():
|
||||
try:
|
||||
parse_cedapug_custom_html("<html></html>")
|
||||
except ValueError as exc:
|
||||
assert "renderCustomMapDownloads" in str(exc)
|
||||
else:
|
||||
raise AssertionError("missing embedded data must fail")
|
||||
89
l4d2web/tests/test_global_overlay_builders.py
Normal file
89
l4d2web/tests/test_global_overlay_builders.py
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from l4d2web.db import init_db, session_scope
|
||||
from l4d2web.models import GlobalOverlayItem, GlobalOverlayItemFile, GlobalOverlaySource, Overlay
|
||||
from l4d2web.services.overlay_builders import BUILDERS
|
||||
|
||||
|
||||
def seed_source(tmp_path: Path, monkeypatch) -> int:
|
||||
monkeypatch.setenv("DATABASE_URL", f"sqlite:///{tmp_path/'builder.db'}")
|
||||
monkeypatch.setenv("LEFT4ME_ROOT", str(tmp_path))
|
||||
init_db()
|
||||
cache_vpk = tmp_path / "global_overlay_cache" / "l4d2center-maps" / "vpks" / "carriedoff.vpk"
|
||||
cache_vpk.parent.mkdir(parents=True, exist_ok=True)
|
||||
cache_vpk.write_bytes(b"vpk")
|
||||
with session_scope() as db:
|
||||
overlay = Overlay(name="l4d2center-maps", path="7", type="l4d2center_maps", user_id=None)
|
||||
db.add(overlay)
|
||||
db.flush()
|
||||
source = GlobalOverlaySource(
|
||||
overlay_id=overlay.id,
|
||||
source_key="l4d2center-maps",
|
||||
source_type="l4d2center_csv",
|
||||
source_url="https://l4d2center.com/maps/servers/index.csv",
|
||||
)
|
||||
db.add(source)
|
||||
db.flush()
|
||||
item = GlobalOverlayItem(
|
||||
source_id=source.id,
|
||||
item_key="carriedoff.vpk",
|
||||
display_name="carriedoff.vpk",
|
||||
download_url="https://example.invalid/carriedoff.7z",
|
||||
expected_vpk_name="carriedoff.vpk",
|
||||
)
|
||||
db.add(item)
|
||||
db.flush()
|
||||
db.add(
|
||||
GlobalOverlayItemFile(
|
||||
item_id=item.id,
|
||||
vpk_name="carriedoff.vpk",
|
||||
cache_path="l4d2center-maps/vpks/carriedoff.vpk",
|
||||
size=3,
|
||||
md5="",
|
||||
)
|
||||
)
|
||||
db.flush()
|
||||
return overlay.id
|
||||
|
||||
|
||||
def test_registry_contains_global_map_builders():
|
||||
assert "l4d2center_maps" in BUILDERS
|
||||
assert "cedapug_maps" in BUILDERS
|
||||
|
||||
|
||||
def test_global_builder_creates_absolute_symlink(tmp_path, monkeypatch):
|
||||
overlay_id = seed_source(tmp_path, monkeypatch)
|
||||
out: list[str] = []
|
||||
err: list[str] = []
|
||||
with session_scope() as db:
|
||||
overlay = db.query(Overlay).filter_by(id=overlay_id).one()
|
||||
BUILDERS["l4d2center_maps"].build(overlay, on_stdout=out.append, on_stderr=err.append, should_cancel=lambda: False)
|
||||
|
||||
link = tmp_path / "overlays" / "7" / "left4dead2" / "addons" / "carriedoff.vpk"
|
||||
assert link.is_symlink()
|
||||
assert os.path.isabs(os.readlink(link))
|
||||
assert link.resolve() == (tmp_path / "global_overlay_cache" / "l4d2center-maps" / "vpks" / "carriedoff.vpk").resolve()
|
||||
assert any("global overlay" in line for line in out)
|
||||
|
||||
|
||||
def test_global_builder_removes_obsolete_managed_symlink_but_keeps_foreign(tmp_path, monkeypatch):
|
||||
overlay_id = seed_source(tmp_path, monkeypatch)
|
||||
addons = tmp_path / "overlays" / "7" / "left4dead2" / "addons"
|
||||
addons.mkdir(parents=True, exist_ok=True)
|
||||
foreign_target = tmp_path / "foreign.vpk"
|
||||
foreign_target.write_bytes(b"foreign")
|
||||
os.symlink(str(foreign_target), addons / "foreign.vpk")
|
||||
|
||||
with session_scope() as db:
|
||||
overlay = db.query(Overlay).filter_by(id=overlay_id).one()
|
||||
BUILDERS["l4d2center_maps"].build(overlay, on_stdout=lambda line: None, on_stderr=lambda line: None, should_cancel=lambda: False)
|
||||
source = db.query(GlobalOverlaySource).filter_by(source_key="l4d2center-maps").one()
|
||||
db.query(GlobalOverlayItem).filter_by(source_id=source.id).delete()
|
||||
|
||||
with session_scope() as db:
|
||||
overlay = db.query(Overlay).filter_by(id=overlay_id).one()
|
||||
BUILDERS["l4d2center_maps"].build(overlay, on_stdout=lambda line: None, on_stderr=lambda line: None, should_cancel=lambda: False)
|
||||
|
||||
assert not (addons / "carriedoff.vpk").exists()
|
||||
assert (addons / "foreign.vpk").is_symlink()
|
||||
19
l4d2web/tests/test_global_overlay_cli.py
Normal file
19
l4d2web/tests/test_global_overlay_cli.py
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
from l4d2web.app import create_app
|
||||
from l4d2web.db import init_db, session_scope
|
||||
from l4d2web.models import Job
|
||||
|
||||
|
||||
def test_refresh_global_overlays_cli_enqueues_system_job(tmp_path, monkeypatch):
|
||||
db_url = f"sqlite:///{tmp_path/'cli.db'}"
|
||||
monkeypatch.setenv("DATABASE_URL", db_url)
|
||||
monkeypatch.setenv("LEFT4ME_ROOT", str(tmp_path))
|
||||
app = create_app({"TESTING": True, "DATABASE_URL": db_url, "SECRET_KEY": "test"})
|
||||
init_db()
|
||||
|
||||
result = app.test_cli_runner().invoke(args=["refresh-global-overlays"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "queued refresh_global_overlays job" in result.output
|
||||
with session_scope() as db:
|
||||
job = db.query(Job).filter_by(operation="refresh_global_overlays").one()
|
||||
assert job.user_id is None
|
||||
154
l4d2web/tests/test_global_overlay_models.py
Normal file
154
l4d2web/tests/test_global_overlay_models.py
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from l4d2web.db import init_db, session_scope
|
||||
from l4d2web.models import (
|
||||
GlobalOverlayItem,
|
||||
GlobalOverlayItemFile,
|
||||
GlobalOverlaySource,
|
||||
Job,
|
||||
Overlay,
|
||||
User,
|
||||
)
|
||||
|
||||
|
||||
def test_system_job_allows_null_user_id(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DATABASE_URL", f"sqlite:///{tmp_path/'models.db'}")
|
||||
init_db()
|
||||
|
||||
with session_scope() as db:
|
||||
job = Job(
|
||||
user_id=None,
|
||||
server_id=None,
|
||||
overlay_id=None,
|
||||
operation="refresh_global_overlays",
|
||||
)
|
||||
db.add(job)
|
||||
db.flush()
|
||||
assert job.id is not None
|
||||
assert job.user_id is None
|
||||
|
||||
|
||||
def test_global_overlay_source_uniqueness(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DATABASE_URL", f"sqlite:///{tmp_path/'sources.db'}")
|
||||
init_db()
|
||||
|
||||
with session_scope() as db:
|
||||
overlay = Overlay(
|
||||
name="l4d2center-maps", path="1", type="l4d2center_maps", user_id=None
|
||||
)
|
||||
db.add(overlay)
|
||||
db.flush()
|
||||
db.add(
|
||||
GlobalOverlaySource(
|
||||
overlay_id=overlay.id,
|
||||
source_key="l4d2center-maps",
|
||||
source_type="l4d2center_csv",
|
||||
source_url="https://l4d2center.com/maps/servers/index.csv",
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
with session_scope() as db:
|
||||
other = Overlay(
|
||||
name="cedapug-maps", path="2", type="cedapug_maps", user_id=None
|
||||
)
|
||||
db.add(other)
|
||||
db.flush()
|
||||
db.add(
|
||||
GlobalOverlaySource(
|
||||
overlay_id=other.id,
|
||||
source_key="l4d2center-maps",
|
||||
source_type="l4d2center_csv",
|
||||
source_url="https://example.invalid/duplicate",
|
||||
)
|
||||
)
|
||||
except IntegrityError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("duplicate source_key must fail")
|
||||
|
||||
|
||||
def test_global_overlay_items_and_files_are_unique_per_parent(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DATABASE_URL", f"sqlite:///{tmp_path/'items.db'}")
|
||||
init_db()
|
||||
|
||||
with session_scope() as db:
|
||||
overlay = Overlay(name="cedapug-maps", path="1", type="cedapug_maps", user_id=None)
|
||||
db.add(overlay)
|
||||
db.flush()
|
||||
source = GlobalOverlaySource(
|
||||
overlay_id=overlay.id,
|
||||
source_key="cedapug-maps",
|
||||
source_type="cedapug_custom_page",
|
||||
source_url="https://cedapug.com/custom",
|
||||
)
|
||||
db.add(source)
|
||||
db.flush()
|
||||
item = GlobalOverlayItem(
|
||||
source_id=source.id,
|
||||
item_key="FatalFreight.zip",
|
||||
display_name="Fatal Freight",
|
||||
download_url="https://cedapug.com/maps/FatalFreight.zip",
|
||||
expected_vpk_name="FatalFreight.vpk",
|
||||
)
|
||||
db.add(item)
|
||||
db.flush()
|
||||
db.add(
|
||||
GlobalOverlayItemFile(
|
||||
item_id=item.id,
|
||||
vpk_name="FatalFreight.vpk",
|
||||
cache_path="cedapug-maps/vpks/FatalFreight.vpk",
|
||||
size=123,
|
||||
md5="",
|
||||
)
|
||||
)
|
||||
item_id = item.id
|
||||
|
||||
try:
|
||||
with session_scope() as db:
|
||||
source = db.query(GlobalOverlaySource).filter_by(source_key="cedapug-maps").one()
|
||||
db.add(
|
||||
GlobalOverlayItem(
|
||||
source_id=source.id,
|
||||
item_key="FatalFreight.zip",
|
||||
display_name="Fatal Freight duplicate",
|
||||
download_url="https://cedapug.com/maps/FatalFreight.zip",
|
||||
expected_vpk_name="FatalFreight.vpk",
|
||||
)
|
||||
)
|
||||
except IntegrityError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("duplicate item_key per source must fail")
|
||||
|
||||
try:
|
||||
with session_scope() as db:
|
||||
db.add(
|
||||
GlobalOverlayItemFile(
|
||||
item_id=item_id,
|
||||
vpk_name="FatalFreight.vpk",
|
||||
cache_path="cedapug-maps/vpks/FatalFreight-copy.vpk",
|
||||
size=456,
|
||||
md5="",
|
||||
)
|
||||
)
|
||||
except IntegrityError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("duplicate vpk_name per item must fail")
|
||||
|
||||
|
||||
def test_normal_user_rows_still_require_real_users(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DATABASE_URL", f"sqlite:///{tmp_path/'users.db'}")
|
||||
init_db()
|
||||
|
||||
with session_scope() as db:
|
||||
user = User(username="alice", password_digest="digest", admin=False)
|
||||
db.add(user)
|
||||
db.flush()
|
||||
job = Job(user_id=user.id, server_id=None, operation="install", state="queued")
|
||||
db.add(job)
|
||||
db.flush()
|
||||
|
||||
assert job.id is not None
|
||||
assert job.user_id == user.id
|
||||
69
l4d2web/tests/test_global_overlay_refresh.py
Normal file
69
l4d2web/tests/test_global_overlay_refresh.py
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
from pathlib import Path
|
||||
|
||||
from l4d2web.db import init_db, session_scope
|
||||
from l4d2web.models import GlobalOverlayItem, GlobalOverlayItemFile, GlobalOverlaySource
|
||||
from l4d2web.services.global_map_sources import GlobalMapManifestItem
|
||||
|
||||
|
||||
def test_refresh_global_overlays_updates_manifest_items_and_invokes_builders(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DATABASE_URL", f"sqlite:///{tmp_path/'refresh.db'}")
|
||||
monkeypatch.setenv("LEFT4ME_ROOT", str(tmp_path))
|
||||
init_db()
|
||||
|
||||
from l4d2web.services import global_overlay_refresh
|
||||
monkeypatch.setattr(
|
||||
global_overlay_refresh,
|
||||
"fetch_l4d2center_manifest",
|
||||
lambda: ("hash-center", [GlobalMapManifestItem("carriedoff.vpk", "carriedoff.vpk", "https://example.invalid/carriedoff.7z", "carriedoff.vpk", 3, "" )]),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
global_overlay_refresh,
|
||||
"fetch_cedapug_manifest",
|
||||
lambda: ("hash-ceda", [GlobalMapManifestItem("FatalFreight.zip", "Fatal Freight", "https://example.invalid/FatalFreight.zip")]),
|
||||
)
|
||||
|
||||
def fake_download_and_extract(source_key, item, *, should_cancel):
|
||||
target = tmp_path / "global_overlay_cache" / source_key / "vpks" / (item.expected_vpk_name or item.item_key.replace(".zip", ".vpk"))
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(b"vpk")
|
||||
return [(target.name, f"{source_key}/vpks/{target.name}", 3, "")], "etag", "last-modified", 3
|
||||
|
||||
built: list[str] = []
|
||||
monkeypatch.setattr(global_overlay_refresh, "download_and_extract_item", fake_download_and_extract)
|
||||
monkeypatch.setattr(global_overlay_refresh, "build_global_overlay", lambda overlay, **kwargs: built.append(overlay.name))
|
||||
|
||||
out: list[str] = []
|
||||
result = global_overlay_refresh.refresh_global_overlays(on_stdout=out.append, on_stderr=out.append, should_cancel=lambda: False)
|
||||
|
||||
assert result == ["cedapug-maps", "l4d2center-maps"]
|
||||
assert set(built) == {"cedapug-maps", "l4d2center-maps"}
|
||||
with session_scope() as db:
|
||||
assert db.query(GlobalOverlaySource).count() == 2
|
||||
assert db.query(GlobalOverlayItem).count() == 2
|
||||
assert db.query(GlobalOverlayItemFile).count() == 2
|
||||
|
||||
|
||||
def test_refresh_removes_items_absent_from_manifest(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DATABASE_URL", f"sqlite:///{tmp_path/'remove.db'}")
|
||||
monkeypatch.setenv("LEFT4ME_ROOT", str(tmp_path))
|
||||
init_db()
|
||||
|
||||
from l4d2web.services.global_overlays import ensure_global_overlays
|
||||
from l4d2web.services import global_overlay_refresh
|
||||
|
||||
with session_scope() as db:
|
||||
ensure_global_overlays(db)
|
||||
source = db.query(GlobalOverlaySource).filter_by(source_key="l4d2center-maps").one()
|
||||
item = GlobalOverlayItem(source_id=source.id, item_key="old.vpk", display_name="old.vpk", download_url="https://example.invalid/old.7z")
|
||||
db.add(item)
|
||||
db.flush()
|
||||
db.add(GlobalOverlayItemFile(item_id=item.id, vpk_name="old.vpk", cache_path="l4d2center-maps/vpks/old.vpk", size=3))
|
||||
|
||||
monkeypatch.setattr(global_overlay_refresh, "fetch_l4d2center_manifest", lambda: ("empty-center", []))
|
||||
monkeypatch.setattr(global_overlay_refresh, "fetch_cedapug_manifest", lambda: ("empty-ceda", []))
|
||||
monkeypatch.setattr(global_overlay_refresh, "build_global_overlay", lambda overlay, **kwargs: None)
|
||||
|
||||
global_overlay_refresh.refresh_global_overlays(on_stdout=lambda line: None, on_stderr=lambda line: None, should_cancel=lambda: False)
|
||||
|
||||
with session_scope() as db:
|
||||
assert db.query(GlobalOverlayItem).filter_by(item_key="old.vpk").count() == 0
|
||||
167
l4d2web/tests/test_global_overlays.py
Normal file
167
l4d2web/tests/test_global_overlays.py
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
from sqlalchemy import select
|
||||
|
||||
from l4d2web.db import init_db, session_scope
|
||||
from l4d2web.models import GlobalOverlaySource, Job, Overlay, User
|
||||
from l4d2web.services.global_overlays import (
|
||||
enqueue_refresh_global_overlays,
|
||||
ensure_global_overlays,
|
||||
is_creatable_overlay_type,
|
||||
)
|
||||
|
||||
|
||||
def test_ensure_global_overlays_creates_singletons_and_directories(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DATABASE_URL", f"sqlite:///{tmp_path/'global_overlays.db'}")
|
||||
monkeypatch.setenv("LEFT4ME_ROOT", str(tmp_path))
|
||||
init_db()
|
||||
|
||||
with session_scope() as session:
|
||||
created = ensure_global_overlays(session)
|
||||
assert created == {"cedapug-maps", "l4d2center-maps"}
|
||||
|
||||
second = ensure_global_overlays(session)
|
||||
assert second == set()
|
||||
|
||||
overlays = session.scalars(select(Overlay).order_by(Overlay.name)).all()
|
||||
assert [overlay.name for overlay in overlays] == ["cedapug-maps", "l4d2center-maps"]
|
||||
assert [overlay.type for overlay in overlays] == ["cedapug_maps", "l4d2center_maps"]
|
||||
assert [overlay.user_id for overlay in overlays] == [None, None]
|
||||
assert len({overlay.path for overlay in overlays}) == 2
|
||||
for overlay in overlays:
|
||||
assert (tmp_path / "overlays" / overlay.path).is_dir()
|
||||
|
||||
sources = session.scalars(select(GlobalOverlaySource).order_by(GlobalOverlaySource.source_key)).all()
|
||||
assert [source.source_key for source in sources] == ["cedapug-maps", "l4d2center-maps"]
|
||||
assert [source.source_type for source in sources] == [
|
||||
"cedapug_custom_page",
|
||||
"l4d2center_csv",
|
||||
]
|
||||
|
||||
|
||||
def test_ensure_global_overlays_repairs_existing_rows(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DATABASE_URL", f"sqlite:///{tmp_path/'global_overlay_repair.db'}")
|
||||
monkeypatch.setenv("LEFT4ME_ROOT", str(tmp_path))
|
||||
init_db()
|
||||
|
||||
with session_scope() as session:
|
||||
overlay = Overlay(name="cedapug-maps", path="legacy", type="cedapug_maps", user_id=None)
|
||||
session.add(overlay)
|
||||
session.flush()
|
||||
session.add(
|
||||
GlobalOverlaySource(
|
||||
overlay_id=overlay.id,
|
||||
source_key="cedapug-maps",
|
||||
source_type="wrong",
|
||||
source_url="https://example.invalid/wrong",
|
||||
)
|
||||
)
|
||||
|
||||
(tmp_path / "overlays" / "legacy").mkdir(parents=True)
|
||||
|
||||
with session_scope() as session:
|
||||
created = ensure_global_overlays(session)
|
||||
assert created == {"l4d2center-maps"}
|
||||
|
||||
repaired = session.scalar(select(Overlay).where(Overlay.name == "cedapug-maps"))
|
||||
assert repaired is not None
|
||||
assert repaired.type == "cedapug_maps"
|
||||
assert repaired.user_id is None
|
||||
assert (tmp_path / "overlays" / repaired.path).is_dir()
|
||||
|
||||
source = session.scalar(
|
||||
select(GlobalOverlaySource).where(GlobalOverlaySource.source_key == "cedapug-maps")
|
||||
)
|
||||
assert source is not None
|
||||
assert source.source_type == "cedapug_custom_page"
|
||||
assert source.source_url == "https://cedapug.com/custom"
|
||||
|
||||
|
||||
def test_ensure_global_overlays_does_not_hijack_private_overlay_name(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DATABASE_URL", f"sqlite:///{tmp_path/'global_overlay_private_name.db'}")
|
||||
monkeypatch.setenv("LEFT4ME_ROOT", str(tmp_path))
|
||||
init_db()
|
||||
|
||||
with session_scope() as session:
|
||||
user = User(username="alice", password_digest="digest", admin=False)
|
||||
session.add(user)
|
||||
session.flush()
|
||||
private = Overlay(
|
||||
name="l4d2center-maps",
|
||||
path="private-l4d2center",
|
||||
type="workshop",
|
||||
user_id=user.id,
|
||||
)
|
||||
session.add(private)
|
||||
session.flush()
|
||||
private_id = private.id
|
||||
private_user_id = user.id
|
||||
|
||||
with session_scope() as session:
|
||||
created = ensure_global_overlays(session)
|
||||
|
||||
assert created == {"cedapug-maps", "l4d2center-maps"}
|
||||
private = session.scalar(select(Overlay).where(Overlay.id == private_id))
|
||||
assert private is not None
|
||||
assert private.user_id == private_user_id
|
||||
assert private.type == "workshop"
|
||||
assert private.path == "private-l4d2center"
|
||||
|
||||
system = session.scalar(
|
||||
select(Overlay).where(Overlay.name == "l4d2center-maps", Overlay.user_id.is_(None))
|
||||
)
|
||||
assert system is not None
|
||||
assert system.id != private_id
|
||||
assert system.type == "l4d2center_maps"
|
||||
assert (tmp_path / "overlays" / system.path).is_dir()
|
||||
|
||||
source = session.scalar(
|
||||
select(GlobalOverlaySource).where(GlobalOverlaySource.source_key == "l4d2center-maps")
|
||||
)
|
||||
assert source is not None
|
||||
assert source.overlay_id == system.id
|
||||
|
||||
|
||||
def test_enqueue_refresh_global_overlays_coalesces_active_jobs(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DATABASE_URL", f"sqlite:///{tmp_path/'refresh_jobs.db'}")
|
||||
init_db()
|
||||
|
||||
for state in ("queued", "running", "cancelling"):
|
||||
with session_scope() as session:
|
||||
session.query(Job).delete()
|
||||
existing = Job(
|
||||
user_id=7,
|
||||
server_id=None,
|
||||
overlay_id=None,
|
||||
operation="refresh_global_overlays",
|
||||
state=state,
|
||||
)
|
||||
session.add(existing)
|
||||
session.flush()
|
||||
existing_id = existing.id
|
||||
|
||||
job = enqueue_refresh_global_overlays(session, user_id=None)
|
||||
assert job.id == existing_id
|
||||
assert session.query(Job).filter_by(operation="refresh_global_overlays").count() == 1
|
||||
|
||||
|
||||
def test_enqueue_refresh_global_overlays_creates_system_job(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DATABASE_URL", f"sqlite:///{tmp_path/'refresh_system_job.db'}")
|
||||
init_db()
|
||||
|
||||
with session_scope() as session:
|
||||
job = enqueue_refresh_global_overlays(session, user_id=None)
|
||||
|
||||
assert job.id is not None
|
||||
assert job.user_id is None
|
||||
assert job.server_id is None
|
||||
assert job.overlay_id is None
|
||||
assert job.operation == "refresh_global_overlays"
|
||||
assert job.state == "queued"
|
||||
|
||||
|
||||
def test_is_creatable_overlay_type_policy():
|
||||
assert is_creatable_overlay_type("workshop", admin=False) is True
|
||||
assert is_creatable_overlay_type("workshop", admin=True) is True
|
||||
assert is_creatable_overlay_type("external", admin=False) is False
|
||||
assert is_creatable_overlay_type("external", admin=True) is False
|
||||
assert is_creatable_overlay_type("l4d2center_maps", admin=True) is False
|
||||
assert is_creatable_overlay_type("cedapug_maps", admin=True) is False
|
||||
|
|
@ -117,7 +117,7 @@ def test_system_job_logs_persist(tmp_path, monkeypatch):
|
|||
job = Job(
|
||||
user_id=None,
|
||||
server_id=None,
|
||||
operation="refresh_workshop_items",
|
||||
operation="refresh_global_overlays",
|
||||
state="queued",
|
||||
)
|
||||
db.add(job)
|
||||
|
|
|
|||
|
|
@ -564,56 +564,6 @@ def test_run_worker_once_dispatches_build_overlay(overlay_seeded_worker, monkeyp
|
|||
assert (addons / "1001.vpk").is_symlink()
|
||||
|
||||
|
||||
def test_build_overlay_writes_last_build_status_ok(
|
||||
overlay_seeded_worker, monkeypatch, tmp_path
|
||||
) -> None:
|
||||
app, ids = overlay_seeded_worker
|
||||
monkeypatch.setenv("LEFT4ME_ROOT", str(tmp_path))
|
||||
|
||||
from l4d2web.services import overlay_builders
|
||||
|
||||
class _StubBuilder:
|
||||
def build(self, overlay, *, on_stdout, on_stderr, should_cancel):
|
||||
on_stdout("stub build ok")
|
||||
|
||||
monkeypatch.setitem(overlay_builders.BUILDERS, "workshop", _StubBuilder())
|
||||
|
||||
job_id = add_job(ids.user, "build_overlay", server_id=None, overlay_id=ids.overlay)
|
||||
|
||||
with app.app_context():
|
||||
assert run_worker_once() is True
|
||||
|
||||
assert load_job(job_id).state == "succeeded"
|
||||
with session_scope() as s:
|
||||
overlay = s.query(Overlay).filter_by(id=ids.overlay).one()
|
||||
assert overlay.last_build_status == "ok"
|
||||
|
||||
|
||||
def test_build_overlay_writes_last_build_status_failed(
|
||||
overlay_seeded_worker, monkeypatch, tmp_path
|
||||
) -> None:
|
||||
app, ids = overlay_seeded_worker
|
||||
monkeypatch.setenv("LEFT4ME_ROOT", str(tmp_path))
|
||||
|
||||
from l4d2web.services import overlay_builders
|
||||
|
||||
class _FailingBuilder:
|
||||
def build(self, overlay, *, on_stdout, on_stderr, should_cancel):
|
||||
raise RuntimeError("synthetic build failure")
|
||||
|
||||
monkeypatch.setitem(overlay_builders.BUILDERS, "workshop", _FailingBuilder())
|
||||
|
||||
job_id = add_job(ids.user, "build_overlay", server_id=None, overlay_id=ids.overlay)
|
||||
|
||||
with app.app_context():
|
||||
assert run_worker_once() is True
|
||||
|
||||
assert load_job(job_id).state == "failed"
|
||||
with session_scope() as s:
|
||||
overlay = s.query(Overlay).filter_by(id=ids.overlay).one()
|
||||
assert overlay.last_build_status == "failed"
|
||||
|
||||
|
||||
def test_run_worker_once_dispatches_refresh(overlay_seeded_worker, monkeypatch, tmp_path) -> None:
|
||||
app, ids = overlay_seeded_worker
|
||||
monkeypatch.setenv("LEFT4ME_ROOT", str(tmp_path))
|
||||
|
|
@ -752,29 +702,46 @@ def test_refresh_job_enqueues_build_overlay_without_locking_its_final_log(
|
|||
assert "enqueued build_overlay for 1 overlay(s)" in lines
|
||||
|
||||
|
||||
def test_global_operations_set() -> None:
|
||||
from l4d2web.services.job_worker import GLOBAL_OPERATIONS
|
||||
def test_refresh_global_overlays_blocks_install_build_refresh_and_servers() -> None:
|
||||
from l4d2web.services.job_worker import SchedulerState, can_start
|
||||
|
||||
assert GLOBAL_OPERATIONS == {"install", "refresh_workshop_items"}
|
||||
state = SchedulerState(refresh_global_overlays_running=True)
|
||||
assert can_start(DummyJob(operation="install"), state) is False
|
||||
assert can_start(DummyJob(operation="refresh_workshop_items"), state) is False
|
||||
assert can_start(DummyJob(operation="build_overlay", overlay_id=1), state) is False
|
||||
assert can_start(DummyJob(operation="start", server_id=1), state) is False
|
||||
|
||||
|
||||
def test_build_overlay_script_type_blocks_per_overlay(overlay_seeded_worker) -> None:
|
||||
"""Mechanically identical to workshop builds, but pinned for script type."""
|
||||
app, ids = overlay_seeded_worker
|
||||
with session_scope() as s:
|
||||
overlay = s.query(Overlay).filter_by(id=ids.overlay).one()
|
||||
overlay.type = "script"
|
||||
overlay.script = "echo hi"
|
||||
|
||||
add_job(ids.user, "build_overlay", server_id=None, state="running", overlay_id=ids.overlay)
|
||||
def test_refresh_global_overlays_waits_for_active_work() -> None:
|
||||
from l4d2web.services.job_worker import SchedulerState, can_start
|
||||
|
||||
assert can_start(DummyJob(operation="refresh_global_overlays"), SchedulerState(install_running=True)) is False
|
||||
assert can_start(DummyJob(operation="refresh_global_overlays"), SchedulerState(refresh_running=True)) is False
|
||||
state = SchedulerState()
|
||||
state.running_overlays.add(ids.overlay)
|
||||
assert (
|
||||
can_start(DummyJob(operation="build_overlay", overlay_id=ids.overlay), state)
|
||||
is False
|
||||
)
|
||||
assert (
|
||||
can_start(DummyJob(operation="build_overlay", overlay_id=ids.overlay + 1), state)
|
||||
is True
|
||||
)
|
||||
state.running_overlays.add(1)
|
||||
assert can_start(DummyJob(operation="refresh_global_overlays"), state) is False
|
||||
state = SchedulerState()
|
||||
state.running_servers.add(1)
|
||||
assert can_start(DummyJob(operation="refresh_global_overlays"), state) is False
|
||||
|
||||
|
||||
def test_run_worker_once_dispatches_refresh_global_overlays(seeded_worker, monkeypatch):
|
||||
from l4d2web.services import job_worker
|
||||
from l4d2web.models import Job
|
||||
from l4d2web.db import session_scope
|
||||
|
||||
called = []
|
||||
|
||||
def fake_refresh(*, on_stdout, on_stderr, should_cancel):
|
||||
called.append("refresh")
|
||||
on_stdout("global refresh complete")
|
||||
return ["l4d2center-maps"]
|
||||
|
||||
monkeypatch.setattr(job_worker, "_run_refresh_global_overlays", fake_refresh)
|
||||
with session_scope() as db:
|
||||
job = Job(user_id=None, server_id=None, operation="refresh_global_overlays", state="queued")
|
||||
db.add(job)
|
||||
|
||||
app, ids = seeded_worker
|
||||
assert job_worker.run_worker_once() is True
|
||||
assert called == ["refresh"]
|
||||
|
|
|
|||
|
|
@ -260,3 +260,53 @@ def test_initialize_fails_fast_on_uncached_workshop_items(
|
|||
assert all("initialize" not in cmd for cmd in invocations), invocations
|
||||
|
||||
|
||||
def test_initialize_fails_when_global_overlay_cache_file_missing(tmp_path, monkeypatch):
|
||||
from l4d2web.db import init_db, session_scope
|
||||
from l4d2web.models import (
|
||||
Blueprint,
|
||||
BlueprintOverlay,
|
||||
GlobalOverlayItem,
|
||||
GlobalOverlayItemFile,
|
||||
GlobalOverlaySource,
|
||||
Overlay,
|
||||
Server,
|
||||
User,
|
||||
)
|
||||
from l4d2web.services.l4d2_facade import initialize_server
|
||||
|
||||
monkeypatch.setenv("DATABASE_URL", f"sqlite:///{tmp_path/'facade-global.db'}")
|
||||
monkeypatch.setenv("LEFT4ME_ROOT", str(tmp_path))
|
||||
init_db()
|
||||
|
||||
with session_scope() as db:
|
||||
user = User(username="alice", password_digest="digest")
|
||||
db.add(user)
|
||||
db.flush()
|
||||
overlay = Overlay(name="l4d2center-maps", path="7", type="l4d2center_maps", user_id=None)
|
||||
db.add(overlay)
|
||||
db.flush()
|
||||
source = GlobalOverlaySource(overlay_id=overlay.id, source_key="l4d2center-maps", source_type="l4d2center_csv", source_url="https://l4d2center.com/maps/servers/index.csv")
|
||||
db.add(source)
|
||||
db.flush()
|
||||
item = GlobalOverlayItem(source_id=source.id, item_key="carriedoff.vpk", display_name="carriedoff.vpk", download_url="https://example.invalid/carriedoff.7z")
|
||||
db.add(item)
|
||||
db.flush()
|
||||
db.add(GlobalOverlayItemFile(item_id=item.id, vpk_name="carriedoff.vpk", cache_path="l4d2center-maps/vpks/carriedoff.vpk", size=123))
|
||||
blueprint = Blueprint(user_id=user.id, name="bp", arguments="[]", config="[]")
|
||||
db.add(blueprint)
|
||||
db.flush()
|
||||
db.add(BlueprintOverlay(blueprint_id=blueprint.id, overlay_id=overlay.id, position=0))
|
||||
server = Server(user_id=user.id, blueprint_id=blueprint.id, name="alpha", port=27015)
|
||||
db.add(server)
|
||||
db.flush()
|
||||
server_id = server.id
|
||||
|
||||
monkeypatch.setattr("l4d2web.services.host_commands.run_command", lambda *args, **kwargs: None)
|
||||
|
||||
try:
|
||||
initialize_server(server_id)
|
||||
except RuntimeError as exc:
|
||||
assert "carriedoff.vpk" in str(exc)
|
||||
assert "l4d2center-maps" in str(exc)
|
||||
else:
|
||||
raise AssertionError("missing global overlay cache file must fail")
|
||||
|
|
|
|||
|
|
@ -1,17 +1,15 @@
|
|||
"""Tests for overlay builders (registry, WorkshopBuilder, ScriptBuilder)."""
|
||||
"""Tests for overlay builders (registry, WorkshopBuilder)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from l4d2web.db import init_db, session_scope
|
||||
from l4d2web.models import Overlay, OverlayWorkshopItem, User, WorkshopItem
|
||||
from l4d2web.services import overlay_builders
|
||||
from l4d2web.services.host_commands import CommandCancelledError, CommandResult
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -63,13 +61,9 @@ def _capture_logs():
|
|||
return out, err, out.append, err.append
|
||||
|
||||
|
||||
def test_builders_registry() -> None:
|
||||
assert set(overlay_builders.BUILDERS) == {"workshop", "script"}
|
||||
|
||||
|
||||
def test_registry_excludes_legacy_types() -> None:
|
||||
for legacy in ("external", "l4d2center_maps", "cedapug_maps"):
|
||||
assert legacy not in overlay_builders.BUILDERS
|
||||
def test_registry_has_workshop() -> None:
|
||||
assert "workshop" in overlay_builders.BUILDERS
|
||||
assert "external" not in overlay_builders.BUILDERS
|
||||
|
||||
|
||||
def test_registry_unknown_type_raises_keyerror() -> None:
|
||||
|
|
@ -77,13 +71,6 @@ def test_registry_unknown_type_raises_keyerror() -> None:
|
|||
overlay_builders.BUILDERS["nope"]
|
||||
|
||||
|
||||
def test_workshop_builder_unchanged() -> None:
|
||||
"""Regression guard against accidental removal during refactor."""
|
||||
builder = overlay_builders.BUILDERS["workshop"]
|
||||
assert isinstance(builder, overlay_builders.WorkshopBuilder)
|
||||
assert hasattr(builder, "build")
|
||||
|
||||
|
||||
def test_workshop_builder_creates_absolute_symlinks(env: Path) -> None:
|
||||
_, overlay_id = _create_user_and_overlay("ws", "workshop")
|
||||
cache_root = env / "workshop_cache"
|
||||
|
|
@ -227,133 +214,3 @@ def test_workshop_builder_honors_should_cancel(env: Path) -> None:
|
|||
overlay_builders.BUILDERS["workshop"].build(
|
||||
overlay, on_stdout=lambda _x: None, on_stderr=lambda _x: None, should_cancel=cancel
|
||||
)
|
||||
|
||||
|
||||
# --- ScriptBuilder ---------------------------------------------------------
|
||||
|
||||
def _script_overlay(*, id_: int = 42, script: str = "echo hi") -> SimpleNamespace:
|
||||
return SimpleNamespace(id=id_, type="script", path=str(id_), script=script)
|
||||
|
||||
|
||||
def test_script_builder_invokes_helper(env, monkeypatch) -> None:
|
||||
captured: dict = {}
|
||||
|
||||
def fake_run(cmd, *, on_stdout, on_stderr, should_cancel):
|
||||
captured["cmd"] = list(cmd)
|
||||
captured["script_text"] = open(cmd[-1]).read()
|
||||
captured["script_path_existed"] = os.path.exists(cmd[-1])
|
||||
return CommandResult(returncode=0, stdout="", stderr="")
|
||||
|
||||
monkeypatch.setattr(overlay_builders, "run_command", fake_run)
|
||||
monkeypatch.setattr(
|
||||
overlay_builders.ScriptBuilder, "_enforce_disk_budget", lambda *a, **kw: None
|
||||
)
|
||||
|
||||
overlay = _script_overlay()
|
||||
overlay_builders.ScriptBuilder().build(
|
||||
overlay,
|
||||
on_stdout=lambda _x: None,
|
||||
on_stderr=lambda _x: None,
|
||||
should_cancel=lambda: False,
|
||||
)
|
||||
|
||||
assert captured["cmd"][:4] == [
|
||||
"sudo",
|
||||
"-n",
|
||||
"/usr/local/libexec/left4me/left4me-script-sandbox",
|
||||
"42",
|
||||
]
|
||||
assert captured["script_text"] == "echo hi"
|
||||
assert captured["script_path_existed"] is True
|
||||
# Tmpfile is unlinked after build.
|
||||
assert not os.path.exists(captured["cmd"][-1])
|
||||
|
||||
|
||||
def test_script_builder_disk_cap(env, monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
overlay_builders,
|
||||
"run_command",
|
||||
lambda *a, **kw: CommandResult(returncode=0, stdout="", stderr=""),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
overlay_builders.subprocess,
|
||||
"check_output",
|
||||
lambda *a, **kw: b"25000000000\t/some/path\n",
|
||||
)
|
||||
|
||||
err: list[str] = []
|
||||
overlay = _script_overlay(script="")
|
||||
|
||||
with pytest.raises(overlay_builders.BuildError):
|
||||
overlay_builders.ScriptBuilder().build(
|
||||
overlay,
|
||||
on_stdout=lambda _x: None,
|
||||
on_stderr=err.append,
|
||||
should_cancel=lambda: False,
|
||||
)
|
||||
|
||||
assert any("20" in line and "GB" in line for line in err), err
|
||||
|
||||
|
||||
def test_script_builder_streams_output(env, monkeypatch) -> None:
|
||||
def fake_run(cmd, *, on_stdout, on_stderr, should_cancel):
|
||||
on_stdout("hello")
|
||||
on_stderr("warn")
|
||||
return CommandResult(returncode=0, stdout="hello", stderr="warn")
|
||||
|
||||
monkeypatch.setattr(overlay_builders, "run_command", fake_run)
|
||||
monkeypatch.setattr(
|
||||
overlay_builders.ScriptBuilder, "_enforce_disk_budget", lambda *a, **kw: None
|
||||
)
|
||||
|
||||
out: list[str] = []
|
||||
err: list[str] = []
|
||||
overlay = _script_overlay(script="")
|
||||
overlay_builders.ScriptBuilder().build(
|
||||
overlay, on_stdout=out.append, on_stderr=err.append, should_cancel=lambda: False
|
||||
)
|
||||
assert out == ["hello"]
|
||||
assert err == ["warn"]
|
||||
|
||||
|
||||
def test_script_builder_passes_should_cancel_through(env, monkeypatch) -> None:
|
||||
captured: dict = {}
|
||||
|
||||
def fake_run(cmd, *, on_stdout, on_stderr, should_cancel):
|
||||
captured["should_cancel"] = should_cancel
|
||||
raise CommandCancelledError(returncode=1, cmd=cmd, output="", stderr="")
|
||||
|
||||
monkeypatch.setattr(overlay_builders, "run_command", fake_run)
|
||||
monkeypatch.setattr(
|
||||
overlay_builders.ScriptBuilder, "_enforce_disk_budget", lambda *a, **kw: None
|
||||
)
|
||||
|
||||
overlay = _script_overlay(script="")
|
||||
with pytest.raises(CommandCancelledError):
|
||||
overlay_builders.ScriptBuilder().build(
|
||||
overlay,
|
||||
on_stdout=lambda _x: None,
|
||||
on_stderr=lambda _x: None,
|
||||
should_cancel=lambda: True,
|
||||
)
|
||||
assert captured["should_cancel"]() is True
|
||||
|
||||
|
||||
def test_script_builder_cleans_up_tmpfile_on_failure(env, monkeypatch) -> None:
|
||||
captured: dict = {}
|
||||
|
||||
def fake_run(cmd, *, on_stdout, on_stderr, should_cancel):
|
||||
captured["script_path"] = cmd[-1]
|
||||
raise CommandCancelledError(returncode=1, cmd=cmd, output="", stderr="")
|
||||
|
||||
monkeypatch.setattr(overlay_builders, "run_command", fake_run)
|
||||
|
||||
overlay = _script_overlay(script="")
|
||||
with pytest.raises(CommandCancelledError):
|
||||
overlay_builders.ScriptBuilder().build(
|
||||
overlay,
|
||||
on_stdout=lambda _x: None,
|
||||
on_stderr=lambda _x: None,
|
||||
should_cancel=lambda: False,
|
||||
)
|
||||
assert not os.path.exists(captured["script_path"])
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import pytest
|
|||
from l4d2web.app import create_app
|
||||
from l4d2web.auth import hash_password
|
||||
from l4d2web.db import init_db, session_scope
|
||||
from l4d2web.models import Blueprint, BlueprintOverlay, Overlay, User
|
||||
from l4d2web.models import Blueprint, BlueprintOverlay, GlobalOverlaySource, Overlay, User
|
||||
from l4d2web.services.security import validate_overlay_ref
|
||||
|
||||
|
||||
|
|
@ -38,9 +38,9 @@ def user_client_with_overlay(tmp_path, monkeypatch):
|
|||
with session_scope() as session:
|
||||
user = User(username="alice", password_digest=hash_password("secret"), admin=False)
|
||||
session.add(user)
|
||||
# System overlay (workshop, no user_id), pre-existing.
|
||||
# System overlay (managed-global, no user_id), pre-existing.
|
||||
session.add(
|
||||
Overlay(name="standard", path="standard", type="workshop", user_id=None)
|
||||
Overlay(name="standard", path="standard", type="l4d2center_maps", user_id=None)
|
||||
)
|
||||
session.flush()
|
||||
user_id = user.id
|
||||
|
|
@ -62,6 +62,16 @@ def test_user_can_view_overlay_catalog(user_client_with_overlay) -> None:
|
|||
assert "Create overlay" in text
|
||||
|
||||
|
||||
def test_non_admin_can_view_managed_global_system_overlay(user_client_with_overlay) -> None:
|
||||
_create_managed_global_overlay()
|
||||
|
||||
response = user_client_with_overlay.get("/overlays")
|
||||
text = response.get_data(as_text=True)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "l4d2center-maps" in text
|
||||
|
||||
|
||||
def test_admin_can_view_overlay_edit_controls(admin_client) -> None:
|
||||
response = admin_client.get("/overlays")
|
||||
text = response.get_data(as_text=True)
|
||||
|
|
@ -187,6 +197,62 @@ def test_admin_can_update_and_delete_overlay(admin_client) -> None:
|
|||
assert delete.status_code == 302
|
||||
|
||||
|
||||
def _create_managed_global_overlay() -> int:
|
||||
with session_scope() as session:
|
||||
overlay = Overlay(
|
||||
name="l4d2center-maps",
|
||||
path="managed-l4d2center",
|
||||
type="l4d2center_maps",
|
||||
user_id=None,
|
||||
)
|
||||
session.add(overlay)
|
||||
session.flush()
|
||||
session.add(
|
||||
GlobalOverlaySource(
|
||||
overlay_id=overlay.id,
|
||||
source_key="l4d2center-maps",
|
||||
source_type="l4d2center_csv",
|
||||
source_url="https://l4d2center.com/maps/servers/index.csv",
|
||||
)
|
||||
)
|
||||
return overlay.id
|
||||
|
||||
|
||||
def test_admin_cannot_update_managed_global_overlay(admin_client) -> None:
|
||||
overlay_id = _create_managed_global_overlay()
|
||||
|
||||
response = admin_client.post(
|
||||
f"/overlays/{overlay_id}",
|
||||
data={"name": "renamed"},
|
||||
headers={"X-CSRF-Token": "test-token"},
|
||||
)
|
||||
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
def test_admin_cannot_delete_managed_global_overlay(admin_client) -> None:
|
||||
overlay_id = _create_managed_global_overlay()
|
||||
|
||||
response = admin_client.post(
|
||||
f"/overlays/{overlay_id}/delete",
|
||||
headers={"X-CSRF-Token": "test-token"},
|
||||
)
|
||||
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
def test_admin_overlay_detail_hides_edit_for_managed_global_overlay(admin_client) -> None:
|
||||
overlay_id = _create_managed_global_overlay()
|
||||
|
||||
response = admin_client.get(f"/overlays/{overlay_id}")
|
||||
text = response.get_data(as_text=True)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert f'action="/overlays/{overlay_id}"' not in text
|
||||
assert "delete-overlay-modal" not in text
|
||||
|
||||
|
||||
|
||||
def test_update_overlay_rejects_duplicate_name(admin_client) -> None:
|
||||
ids: list[int] = []
|
||||
for name in ("standard", "competitive"):
|
||||
|
|
@ -239,15 +305,13 @@ def test_overlay_detail_page_lists_using_blueprints(admin_client) -> None:
|
|||
|
||||
|
||||
def test_non_admin_overlay_detail_only_lists_own_using_blueprints(user_client_with_overlay) -> None:
|
||||
overlay_id = _create_managed_global_overlay()
|
||||
with session_scope() as session:
|
||||
alice = session.query(User).filter_by(username="alice").one()
|
||||
other = User(username="mallory", password_digest=hash_password("secret"), admin=False)
|
||||
session.add(other)
|
||||
session.flush()
|
||||
|
||||
# Use the seeded system "standard" overlay (id=1).
|
||||
overlay_id = session.query(Overlay).filter_by(name="standard").one().id
|
||||
|
||||
own_bp = Blueprint(user_id=alice.id, name="own-bp", arguments="[]", config="[]")
|
||||
other_bp = Blueprint(user_id=other.id, name="other-private-bp", arguments="[]", config="[]")
|
||||
session.add_all([own_bp, other_bp])
|
||||
|
|
@ -264,12 +328,12 @@ def test_non_admin_overlay_detail_only_lists_own_using_blueprints(user_client_wi
|
|||
|
||||
|
||||
def test_blueprint_edit_lists_system_and_owned_overlays_only(user_client_with_overlay) -> None:
|
||||
system_overlay_id = _create_managed_global_overlay()
|
||||
with session_scope() as session:
|
||||
alice = session.query(User).filter_by(username="alice").one()
|
||||
other = User(username="mallory", password_digest=hash_password("secret"), admin=False)
|
||||
session.add(other)
|
||||
session.flush()
|
||||
system_overlay_id = session.query(Overlay).filter_by(name="standard").one().id
|
||||
foreign_overlay = Overlay(
|
||||
name="other-private-workshop",
|
||||
path="other-private-workshop",
|
||||
|
|
@ -285,7 +349,7 @@ def test_blueprint_edit_lists_system_and_owned_overlays_only(user_client_with_ov
|
|||
text = response.get_data(as_text=True)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "standard" in text
|
||||
assert "l4d2center-maps" in text
|
||||
assert f'value="{system_overlay_id}"' in text
|
||||
assert "other-private-workshop" not in text
|
||||
|
||||
|
|
@ -295,6 +359,27 @@ def test_overlay_detail_page_404_when_missing(admin_client) -> None:
|
|||
assert response.status_code == 404
|
||||
|
||||
|
||||
def test_overlay_detail_hides_edit_for_non_admin_managed_global(user_client_with_overlay) -> None:
|
||||
# The seeded "standard" managed-global overlay (id=1, user_id=NULL) is read-only for non-admins.
|
||||
response = user_client_with_overlay.get("/overlays/1")
|
||||
text = response.get_data(as_text=True)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "standard" in text
|
||||
assert 'action="/overlays/1"' not in text
|
||||
assert "delete-overlay-modal" not in text
|
||||
|
||||
|
||||
def test_managed_global_overlay_detail_shows_source_url(admin_client) -> None:
|
||||
overlay_id = _create_managed_global_overlay()
|
||||
|
||||
response = admin_client.get(f"/overlays/{overlay_id}")
|
||||
text = response.get_data(as_text=True)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "https://l4d2center.com/maps/servers/index.csv" in text
|
||||
|
||||
|
||||
def test_overlay_update_redirects_to_detail(admin_client) -> None:
|
||||
create = admin_client.post(
|
||||
"/overlays",
|
||||
|
|
@ -337,3 +422,9 @@ def test_delete_overlay_rejects_in_use_overlay(admin_client) -> None:
|
|||
)
|
||||
|
||||
assert response.status_code == 409
|
||||
|
||||
|
||||
def test_admin_can_enqueue_refresh_global_overlays(admin_client):
|
||||
response = admin_client.post("/admin/global-overlays/refresh", headers={"X-CSRF-Token": "test-token"})
|
||||
assert response.status_code == 302
|
||||
assert response.headers["Location"] == "/admin/jobs"
|
||||
|
|
|
|||
|
|
@ -476,13 +476,13 @@ def test_admin_jobs_page_renders_system_job(tmp_path, monkeypatch) -> None:
|
|||
sess["user_id"] = admin_id
|
||||
|
||||
with session_scope() as db:
|
||||
db.add(Job(user_id=None, server_id=None, operation="refresh_workshop_items", state="queued"))
|
||||
db.add(Job(user_id=None, server_id=None, operation="refresh_global_overlays", state="queued"))
|
||||
|
||||
response = admin_client.get("/admin/jobs")
|
||||
text = response.get_data(as_text=True)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "refresh_workshop_items" in text
|
||||
assert "refresh_global_overlays" in text
|
||||
assert "system" in text
|
||||
|
||||
|
||||
|
|
@ -503,68 +503,10 @@ def test_non_admin_cannot_view_system_job(tmp_path, monkeypatch) -> None:
|
|||
sess["user_id"] = user_id
|
||||
|
||||
with session_scope() as db:
|
||||
job = Job(user_id=None, server_id=None, operation="refresh_workshop_items", state="queued")
|
||||
job = Job(user_id=None, server_id=None, operation="refresh_global_overlays", state="queued")
|
||||
db.add(job)
|
||||
db.flush()
|
||||
job_id = job.id
|
||||
|
||||
response = user_client.get(f"/jobs/{job_id}")
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
def test_overlay_create_modal_offers_script_type(auth_client_with_server) -> None:
|
||||
response = auth_client_with_server.get("/overlays")
|
||||
text = response.get_data(as_text=True)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert 'value="workshop"' in text
|
||||
assert 'value="script"' in text
|
||||
|
||||
|
||||
def _seed_overlay(name: str, type_: str, user_id: int) -> int:
|
||||
with session_scope() as s:
|
||||
overlay = Overlay(name=name, path="", type=type_, user_id=user_id)
|
||||
s.add(overlay)
|
||||
s.flush()
|
||||
overlay.path = str(overlay.id)
|
||||
s.flush()
|
||||
return overlay.id
|
||||
|
||||
|
||||
def test_overlay_detail_script_section(auth_client_with_server) -> None:
|
||||
with session_scope() as s:
|
||||
user_id = s.query(User).filter_by(username="alice").one().id
|
||||
overlay_id = _seed_overlay("build", "script", user_id)
|
||||
|
||||
response = auth_client_with_server.get(f"/overlays/{overlay_id}")
|
||||
text = response.get_data(as_text=True)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert '<textarea name="script"' in text
|
||||
assert "Rebuild" in text
|
||||
assert "Wipe" in text
|
||||
assert "Last build" in text
|
||||
|
||||
|
||||
def test_overlay_detail_workshop_section_unchanged(auth_client_with_server) -> None:
|
||||
with session_scope() as s:
|
||||
user_id = s.query(User).filter_by(username="alice").one().id
|
||||
overlay_id = _seed_overlay("ws", "workshop", user_id)
|
||||
|
||||
response = auth_client_with_server.get(f"/overlays/{overlay_id}")
|
||||
text = response.get_data(as_text=True)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "Workshop items" in text
|
||||
|
||||
|
||||
def test_overlay_detail_no_global_source_block(auth_client_with_server) -> None:
|
||||
with session_scope() as s:
|
||||
user_id = s.query(User).filter_by(username="alice").one().id
|
||||
overlay_id = _seed_overlay("ws", "workshop", user_id)
|
||||
|
||||
response = auth_client_with_server.get(f"/overlays/{overlay_id}")
|
||||
text = response.get_data(as_text=True)
|
||||
|
||||
assert "Global source" not in text
|
||||
assert "source_url" not in text
|
||||
|
|
|
|||
|
|
@ -1,256 +0,0 @@
|
|||
"""Routes for type='script' overlays: create, /script (update body),
|
||||
/wipe, /build. Permissions mirror workshop overlays (owner or admin)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from l4d2web.app import create_app
|
||||
from l4d2web.auth import hash_password
|
||||
from l4d2web.db import init_db, session_scope
|
||||
from l4d2web.models import Job, Overlay, User
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app(tmp_path, monkeypatch):
|
||||
db_url = f"sqlite:///{tmp_path/'script-routes.db'}"
|
||||
monkeypatch.setenv("DATABASE_URL", db_url)
|
||||
monkeypatch.setenv("LEFT4ME_ROOT", str(tmp_path))
|
||||
flask_app = create_app({"TESTING": True, "DATABASE_URL": db_url, "SECRET_KEY": "test"})
|
||||
init_db()
|
||||
return flask_app
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def alice_id(app) -> int:
|
||||
with session_scope() as s:
|
||||
user = User(username="alice", password_digest=hash_password("x"), admin=False)
|
||||
s.add(user)
|
||||
s.flush()
|
||||
return user.id
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bob_id(app) -> int:
|
||||
with session_scope() as s:
|
||||
user = User(username="bob", password_digest=hash_password("x"), admin=False)
|
||||
s.add(user)
|
||||
s.flush()
|
||||
return user.id
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def admin_id(app) -> int:
|
||||
with session_scope() as s:
|
||||
user = User(username="admin", password_digest=hash_password("x"), admin=True)
|
||||
s.add(user)
|
||||
s.flush()
|
||||
return user.id
|
||||
|
||||
|
||||
def _client_for(app, user_id: int):
|
||||
client = app.test_client()
|
||||
with client.session_transaction() as sess:
|
||||
sess["user_id"] = user_id
|
||||
sess["csrf_token"] = "test-token"
|
||||
return client
|
||||
|
||||
|
||||
def _create_script_overlay(app, user_id: int, *, name: str = "x") -> int:
|
||||
client = _client_for(app, user_id)
|
||||
response = client.post(
|
||||
"/overlays",
|
||||
data={"name": name, "type": "script"},
|
||||
headers={"X-CSRF-Token": "test-token"},
|
||||
)
|
||||
assert response.status_code == 302, response.get_data(as_text=True)
|
||||
with session_scope() as s:
|
||||
return s.scalar(select(Overlay.id).where(Overlay.name == name))
|
||||
|
||||
|
||||
def test_create_script_overlay(app, alice_id) -> None:
|
||||
client = _client_for(app, alice_id)
|
||||
response = client.post(
|
||||
"/overlays",
|
||||
data={"name": "first", "type": "script"},
|
||||
headers={"X-CSRF-Token": "test-token"},
|
||||
)
|
||||
assert response.status_code == 302
|
||||
with session_scope() as s:
|
||||
overlay = s.query(Overlay).filter_by(name="first").one()
|
||||
assert overlay.type == "script"
|
||||
assert overlay.script == ""
|
||||
assert overlay.last_build_status == ""
|
||||
assert overlay.user_id == alice_id
|
||||
assert overlay.path == str(overlay.id)
|
||||
|
||||
|
||||
def test_admin_creates_system_wide_script_overlay(app, admin_id) -> None:
|
||||
client = _client_for(app, admin_id)
|
||||
response = client.post(
|
||||
"/overlays",
|
||||
data={"name": "system", "type": "script", "system_wide": "1"},
|
||||
headers={"X-CSRF-Token": "test-token"},
|
||||
)
|
||||
assert response.status_code == 302
|
||||
with session_scope() as s:
|
||||
overlay = s.query(Overlay).filter_by(name="system").one()
|
||||
assert overlay.user_id is None
|
||||
|
||||
|
||||
def test_non_admin_system_wide_flag_is_ignored(app, alice_id) -> None:
|
||||
client = _client_for(app, alice_id)
|
||||
response = client.post(
|
||||
"/overlays",
|
||||
data={"name": "evil", "type": "script", "system_wide": "1"},
|
||||
headers={"X-CSRF-Token": "test-token"},
|
||||
)
|
||||
assert response.status_code == 302
|
||||
with session_scope() as s:
|
||||
overlay = s.query(Overlay).filter_by(name="evil").one()
|
||||
assert overlay.user_id == alice_id
|
||||
|
||||
|
||||
def test_update_script_body_enqueues_build(app, alice_id) -> None:
|
||||
overlay_id = _create_script_overlay(app, alice_id)
|
||||
client = _client_for(app, alice_id)
|
||||
|
||||
r1 = client.post(
|
||||
f"/overlays/{overlay_id}/script",
|
||||
data={"script": "echo new"},
|
||||
headers={"X-CSRF-Token": "test-token"},
|
||||
)
|
||||
assert r1.status_code == 302
|
||||
with session_scope() as s:
|
||||
overlay = s.query(Overlay).filter_by(id=overlay_id).one()
|
||||
assert overlay.script == "echo new"
|
||||
jobs = s.query(Job).filter_by(operation="build_overlay", overlay_id=overlay_id).all()
|
||||
assert len(jobs) == 1
|
||||
|
||||
# Coalesce against pending.
|
||||
r2 = client.post(
|
||||
f"/overlays/{overlay_id}/script",
|
||||
data={"script": "echo newer"},
|
||||
headers={"X-CSRF-Token": "test-token"},
|
||||
)
|
||||
assert r2.status_code == 302
|
||||
with session_scope() as s:
|
||||
jobs = s.query(Job).filter_by(operation="build_overlay", overlay_id=overlay_id).all()
|
||||
assert len(jobs) == 1
|
||||
|
||||
|
||||
def test_manual_rebuild(app, alice_id) -> None:
|
||||
overlay_id = _create_script_overlay(app, alice_id)
|
||||
client = _client_for(app, alice_id)
|
||||
|
||||
r1 = client.post(
|
||||
f"/overlays/{overlay_id}/build",
|
||||
headers={"X-CSRF-Token": "test-token"},
|
||||
)
|
||||
assert r1.status_code == 302
|
||||
with session_scope() as s:
|
||||
jobs = s.query(Job).filter_by(operation="build_overlay", overlay_id=overlay_id).all()
|
||||
assert len(jobs) == 1
|
||||
|
||||
# Coalesce.
|
||||
r2 = client.post(
|
||||
f"/overlays/{overlay_id}/build",
|
||||
headers={"X-CSRF-Token": "test-token"},
|
||||
)
|
||||
assert r2.status_code == 302
|
||||
with session_scope() as s:
|
||||
jobs = s.query(Job).filter_by(operation="build_overlay", overlay_id=overlay_id).all()
|
||||
assert len(jobs) == 1
|
||||
|
||||
|
||||
def test_wipe_runs_find_delete(app, alice_id, monkeypatch) -> None:
|
||||
overlay_id = _create_script_overlay(app, alice_id)
|
||||
|
||||
# Pre-set a "successful" status so we can verify wipe resets it.
|
||||
with session_scope() as s:
|
||||
overlay = s.query(Overlay).filter_by(id=overlay_id).one()
|
||||
overlay.last_build_status = "ok"
|
||||
|
||||
captured: dict = {}
|
||||
|
||||
def fake_run(overlay_id_arg, script_text, *, on_stdout, on_stderr, should_cancel):
|
||||
captured["overlay_id"] = overlay_id_arg
|
||||
captured["script"] = script_text
|
||||
|
||||
from l4d2web.services import overlay_builders
|
||||
monkeypatch.setattr(overlay_builders, "run_sandboxed_script", fake_run)
|
||||
|
||||
client = _client_for(app, alice_id)
|
||||
response = client.post(
|
||||
f"/overlays/{overlay_id}/wipe",
|
||||
headers={"X-CSRF-Token": "test-token"},
|
||||
)
|
||||
assert response.status_code == 302
|
||||
assert captured["overlay_id"] == overlay_id
|
||||
assert captured["script"] == "find /overlay -mindepth 1 -delete"
|
||||
|
||||
with session_scope() as s:
|
||||
overlay = s.query(Overlay).filter_by(id=overlay_id).one()
|
||||
assert overlay.last_build_status == ""
|
||||
# Wipe does NOT auto-enqueue a rebuild.
|
||||
jobs = s.query(Job).filter_by(operation="build_overlay", overlay_id=overlay_id).all()
|
||||
assert len(jobs) == 0
|
||||
|
||||
|
||||
def test_wipe_refuses_during_running_build(app, alice_id, monkeypatch) -> None:
|
||||
overlay_id = _create_script_overlay(app, alice_id)
|
||||
|
||||
# Mark a build as running for this overlay.
|
||||
with session_scope() as s:
|
||||
s.add(
|
||||
Job(
|
||||
user_id=alice_id,
|
||||
server_id=None,
|
||||
overlay_id=overlay_id,
|
||||
operation="build_overlay",
|
||||
state="running",
|
||||
)
|
||||
)
|
||||
|
||||
invocations: list = []
|
||||
|
||||
def fake_run(*args, **kwargs):
|
||||
invocations.append((args, kwargs))
|
||||
|
||||
from l4d2web.services import overlay_builders
|
||||
monkeypatch.setattr(overlay_builders, "run_sandboxed_script", fake_run)
|
||||
|
||||
client = _client_for(app, alice_id)
|
||||
response = client.post(
|
||||
f"/overlays/{overlay_id}/wipe",
|
||||
headers={"X-CSRF-Token": "test-token"},
|
||||
)
|
||||
assert response.status_code == 409
|
||||
assert invocations == []
|
||||
|
||||
|
||||
def test_permissions_non_owner_denied(app, alice_id, bob_id) -> None:
|
||||
overlay_id = _create_script_overlay(app, alice_id, name="alice-private")
|
||||
bob = _client_for(app, bob_id)
|
||||
|
||||
r1 = bob.post(
|
||||
f"/overlays/{overlay_id}/script",
|
||||
data={"script": "boom"},
|
||||
headers={"X-CSRF-Token": "test-token"},
|
||||
)
|
||||
assert r1.status_code == 403
|
||||
|
||||
|
||||
def test_permissions_admin_can_edit_any(app, alice_id, admin_id) -> None:
|
||||
overlay_id = _create_script_overlay(app, alice_id, name="alice-private")
|
||||
admin = _client_for(app, admin_id)
|
||||
|
||||
r1 = admin.post(
|
||||
f"/overlays/{overlay_id}/script",
|
||||
data={"script": "echo admin"},
|
||||
headers={"X-CSRF-Token": "test-token"},
|
||||
)
|
||||
assert r1.status_code == 302
|
||||
with session_scope() as s:
|
||||
overlay = s.query(Overlay).filter_by(id=overlay_id).one()
|
||||
assert overlay.script == "echo admin"
|
||||
|
|
@ -38,15 +38,6 @@ def test_overlay_has_type_and_user_id(db) -> None:
|
|||
assert row.user_id is None
|
||||
|
||||
|
||||
def test_overlay_has_script_columns(db) -> None:
|
||||
with session_scope() as s:
|
||||
s.add(Overlay(name="defaulted", path="1"))
|
||||
s.flush()
|
||||
row = s.query(Overlay).filter_by(name="defaulted").one()
|
||||
assert row.script == ""
|
||||
assert row.last_build_status == ""
|
||||
|
||||
|
||||
def test_two_system_overlays_with_same_name_are_rejected(db) -> None:
|
||||
with session_scope() as s:
|
||||
s.add(Overlay(name="shared", path="shared", type="l4d2center_maps", user_id=None))
|
||||
|
|
|
|||
Loading…
Reference in a new issue