From 13bd2e48f6414adeb04ce87b8dd11c39056be805 Mon Sep 17 00:00:00 2001 From: mwiegand Date: Mon, 11 May 2026 22:43:22 +0200 Subject: [PATCH] overlay_builders: add _download_with_retry + _sleep_with_cancel helpers Co-Authored-By: Claude Sonnet 4.6 --- l4d2web/services/overlay_builders.py | 55 ++++++++++ l4d2web/tests/test_overlay_builders.py | 135 +++++++++++++++++++++++++ 2 files changed, 190 insertions(+) diff --git a/l4d2web/services/overlay_builders.py b/l4d2web/services/overlay_builders.py index 8d61b89..958927b 100644 --- a/l4d2web/services/overlay_builders.py +++ b/l4d2web/services/overlay_builders.py @@ -10,9 +10,11 @@ from __future__ import annotations import os import subprocess import tempfile +import time as _time from pathlib import Path from typing import Callable, Protocol +import requests as _requests from sqlalchemy import select from l4d2host.paths import get_left4me_root @@ -20,6 +22,7 @@ from l4d2host.paths import get_left4me_root from l4d2web.db import session_scope from l4d2web.models import Overlay, OverlayWorkshopItem, WorkshopItem from l4d2web.services.host_commands import run_command +from l4d2web.services.steam_workshop import WorkshopMetadata, download_to_cache from l4d2web.services.workshop_paths import cache_path, workshop_cache_root @@ -30,6 +33,58 @@ LogSink = Callable[[str], None] SCRIPT_SANDBOX_HELPER = "/usr/local/libexec/left4me/left4me-script-sandbox" DISK_BUDGET_BYTES = 20 * 1024**3 +DOWNLOAD_RETRY_ATTEMPTS = 3 +DOWNLOAD_RETRY_BACKOFF_SECONDS = (1.0, 2.0, 4.0) + + +def _sleep_with_cancel( + seconds: float, + should_cancel: CancelCheck, + *, + poll_interval: float = 0.25, +) -> bool: + """Sleep up to `seconds`, returning early (True) if `should_cancel` becomes + True. Returns False on a full uninterrupted sleep. Polls every + `poll_interval` seconds.""" + deadline = _time.monotonic() + seconds + while True: + if should_cancel(): + return True + remaining = deadline - _time.monotonic() + if remaining <= 0: + return False + _time.sleep(min(poll_interval, remaining)) + + +def _download_with_retry( + meta: WorkshopMetadata, + cache_root: Path, + *, + on_stderr: LogSink, + should_cancel: CancelCheck, +) -> None: + """Wrap `download_to_cache` with bounded retries and cancel-aware backoff. + Raises the last exception after `DOWNLOAD_RETRY_ATTEMPTS` failures. + Raises `InterruptedError` if cancelled during a backoff sleep.""" + last_exc: BaseException | None = None + for attempt in range(1, DOWNLOAD_RETRY_ATTEMPTS + 1): + try: + download_to_cache(meta, cache_root, should_cancel=should_cancel) + return + except InterruptedError: + raise + except (_requests.RequestException, OSError) as exc: + last_exc = exc + if attempt == DOWNLOAD_RETRY_ATTEMPTS: + raise + on_stderr( + f"workshop {meta.steam_id} attempt {attempt}/" + f"{DOWNLOAD_RETRY_ATTEMPTS} failed: {exc}" + ) + delay = DOWNLOAD_RETRY_BACKOFF_SECONDS[attempt - 1] + if _sleep_with_cancel(delay, should_cancel): + raise InterruptedError("download cancelled during backoff") from last_exc + def _sandbox_script_dir() -> Path: """Where script tmpfiles live before being bind-mounted into the sandbox. diff --git a/l4d2web/tests/test_overlay_builders.py b/l4d2web/tests/test_overlay_builders.py index e687f69..9b6c11a 100644 --- a/l4d2web/tests/test_overlay_builders.py +++ b/l4d2web/tests/test_overlay_builders.py @@ -380,3 +380,138 @@ def test_script_builder_cleans_up_tmpfile_on_failure(env, monkeypatch) -> None: should_cancel=lambda: False, ) assert not os.path.exists(captured["script_path"]) + + +def test_sleep_with_cancel_returns_normally_when_not_cancelled(): + from l4d2web.services.overlay_builders import _sleep_with_cancel + + cancelled = _sleep_with_cancel(0.05, lambda: False, poll_interval=0.01) + assert cancelled is False + + +def test_sleep_with_cancel_returns_early_when_cancelled(): + import time + from l4d2web.services.overlay_builders import _sleep_with_cancel + + flag = {"cancel": False} + def cancel_check(): + return flag["cancel"] + + import threading + threading.Timer(0.05, lambda: flag.update(cancel=True)).start() + + start = time.monotonic() + cancelled = _sleep_with_cancel(5.0, cancel_check, poll_interval=0.01) + elapsed = time.monotonic() - start + + assert cancelled is True + assert elapsed < 0.5, f"should have woken up promptly, slept {elapsed:.3f}s" + + +def test_download_with_retry_succeeds_on_first_attempt(env, tmp_path, monkeypatch): + from l4d2web.services import overlay_builders, steam_workshop + + monkeypatch.setattr(overlay_builders, "_sleep_with_cancel", lambda *a, **kw: False) + calls = [] + def fake_download(meta, cache_root, *, should_cancel=None): + calls.append(1) + return cache_root / f"{meta.steam_id}.vpk" + monkeypatch.setattr(overlay_builders, "download_to_cache", fake_download) + + meta = steam_workshop.WorkshopMetadata( + steam_id="1001", title="x", filename="x.vpk", file_url="http://e/x.vpk", + file_size=1, time_updated=1700000000, preview_url="", consumer_app_id=550, result=1, + ) + out, err, on_stdout, on_stderr = _capture_logs() + overlay_builders._download_with_retry( + meta, tmp_path / "cache", + on_stderr=on_stderr, should_cancel=lambda: False, + ) + assert calls == [1] + assert err == [] + + +def test_download_with_retry_retries_then_succeeds(env, tmp_path, monkeypatch): + import requests + from l4d2web.services import overlay_builders, steam_workshop + + monkeypatch.setattr(overlay_builders, "_sleep_with_cancel", lambda *a, **kw: False) + attempts = {"n": 0} + def fake_download(meta, cache_root, *, should_cancel=None): + attempts["n"] += 1 + if attempts["n"] < 3: + raise requests.ConnectionError("boom") + monkeypatch.setattr(overlay_builders, "download_to_cache", fake_download) + + meta = steam_workshop.WorkshopMetadata( + steam_id="1001", title="x", filename="x.vpk", file_url="http://e/x.vpk", + file_size=1, time_updated=1700000000, preview_url="", consumer_app_id=550, result=1, + ) + out, err, on_stdout, on_stderr = _capture_logs() + overlay_builders._download_with_retry( + meta, tmp_path / "cache", + on_stderr=on_stderr, should_cancel=lambda: False, + ) + assert attempts["n"] == 3 + assert sum(1 for line in err if "attempt" in line and "failed" in line) == 2 + + +def test_download_with_retry_exhausts_and_raises(env, tmp_path, monkeypatch): + import requests + from l4d2web.services import overlay_builders, steam_workshop + + monkeypatch.setattr(overlay_builders, "_sleep_with_cancel", lambda *a, **kw: False) + def fake_download(meta, cache_root, *, should_cancel=None): + raise requests.ConnectionError("permanent") + monkeypatch.setattr(overlay_builders, "download_to_cache", fake_download) + + meta = steam_workshop.WorkshopMetadata( + steam_id="1001", title="x", filename="x.vpk", file_url="http://e/x.vpk", + file_size=1, time_updated=1700000000, preview_url="", consumer_app_id=550, result=1, + ) + out, err, on_stdout, on_stderr = _capture_logs() + with pytest.raises(requests.ConnectionError): + overlay_builders._download_with_retry( + meta, tmp_path / "cache", + on_stderr=on_stderr, should_cancel=lambda: False, + ) + + +def test_download_with_retry_propagates_interrupted(env, tmp_path, monkeypatch): + from l4d2web.services import overlay_builders, steam_workshop + + def fake_download(meta, cache_root, *, should_cancel=None): + raise InterruptedError("cancelled") + monkeypatch.setattr(overlay_builders, "download_to_cache", fake_download) + + meta = steam_workshop.WorkshopMetadata( + steam_id="1001", title="x", filename="x.vpk", file_url="http://e/x.vpk", + file_size=1, time_updated=1700000000, preview_url="", consumer_app_id=550, result=1, + ) + out, err, on_stdout, on_stderr = _capture_logs() + with pytest.raises(InterruptedError): + overlay_builders._download_with_retry( + meta, tmp_path / "cache", + on_stderr=on_stderr, should_cancel=lambda: False, + ) + + +def test_download_with_retry_bails_when_cancelled_during_backoff(env, tmp_path, monkeypatch): + import requests + from l4d2web.services import overlay_builders, steam_workshop + + def fake_download(meta, cache_root, *, should_cancel=None): + raise requests.ConnectionError("boom") + monkeypatch.setattr(overlay_builders, "download_to_cache", fake_download) + monkeypatch.setattr(overlay_builders, "_sleep_with_cancel", lambda *a, **kw: True) + + meta = steam_workshop.WorkshopMetadata( + steam_id="1001", title="x", filename="x.vpk", file_url="http://e/x.vpk", + file_size=1, time_updated=1700000000, preview_url="", consumer_app_id=550, result=1, + ) + out, err, on_stdout, on_stderr = _capture_logs() + with pytest.raises(InterruptedError): + overlay_builders._download_with_retry( + meta, tmp_path / "cache", + on_stderr=on_stderr, should_cancel=lambda: False, + )