From 4f78574edd700b908ca4a617ef97ab62c8884459 Mon Sep 17 00:00:00 2001 From: mwiegand Date: Thu, 7 May 2026 17:31:12 +0200 Subject: [PATCH] fix(l4d2-web): keep workshop refresh responsive Limit workshop refresh downloads to one worker and commit build-overlay enqueue work before writing the final job log so SQLite locks do not wedge the web process. --- l4d2web/services/job_worker.py | 13 ++-- l4d2web/tests/test_job_worker.py | 113 +++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 5 deletions(-) diff --git a/l4d2web/services/job_worker.py b/l4d2web/services/job_worker.py index 3afa47e..df4470e 100644 --- a/l4d2web/services/job_worker.py +++ b/l4d2web/services/job_worker.py @@ -28,6 +28,7 @@ ACTIVE_JOB_STATES = {"running", "cancelling"} SERVER_OPERATIONS = {"initialize", "start", "stop", "delete"} OVERLAY_OPERATIONS = {"build_overlay"} GLOBAL_OPERATIONS = {"install", "refresh_workshop_items"} +WORKSHOP_REFRESH_DOWNLOAD_WORKERS = 1 _claim_lock = threading.Lock() _log_lock = threading.RLock() @@ -434,7 +435,10 @@ def _run_refresh_workshop_items( on_stdout(f"downloading {len(download_metas)} items") if download_metas: report = steam_workshop.refresh_all( - download_metas, workshop_cache_root(), should_cancel=should_cancel + download_metas, + workshop_cache_root(), + executor_workers=WORKSHOP_REFRESH_DOWNLOAD_WORKERS, + should_cancel=should_cancel, ) on_stdout( f"download phase complete (downloaded={report.downloaded} errors={report.errors})" @@ -479,10 +483,9 @@ def _run_refresh_workshop_items( if user_id is None: continue enqueue_build_overlay(db, overlay_id=ov_id, user_id=user_id) - on_stdout( - f"enqueued build_overlay for {len(affected_overlay_ids)} overlay(s)" - ) - return list(affected_overlay_ids) + + on_stdout(f"enqueued build_overlay for {len(affected_overlay_ids)} overlay(s)") + return list(affected_overlay_ids) def finish_job(job_id: int, state: str, exit_code: int | None, error: str = "") -> None: diff --git a/l4d2web/tests/test_job_worker.py b/l4d2web/tests/test_job_worker.py index 70f61a0..d6bbc0e 100644 --- a/l4d2web/tests/test_job_worker.py +++ b/l4d2web/tests/test_job_worker.py @@ -587,3 +587,116 @@ def test_run_worker_once_dispatches_refresh(overlay_seeded_worker, monkeypatch, assert refresh_calls == [True] job = load_job(job_id) assert job.state == "succeeded" + + +def test_refresh_downloads_serially_to_keep_web_worker_responsive( + overlay_seeded_worker, monkeypatch, tmp_path +) -> None: + app, ids = overlay_seeded_worker + monkeypatch.setenv("LEFT4ME_ROOT", str(tmp_path)) + + from l4d2web.services import job_worker, steam_workshop + + with session_scope() as s: + wi = WorkshopItem( + steam_id="1001", + title="A", + filename="old.vpk", + file_url="https://example.com/old.vpk", + file_size=1, + time_updated=1, + ) + s.add(wi) + s.flush() + s.add(OverlayWorkshopItem(overlay_id=ids.overlay, workshop_item_id=wi.id)) + + meta = steam_workshop.WorkshopMetadata( + steam_id="1001", + title="A", + filename="new.vpk", + file_url="https://example.com/new.vpk", + file_size=2, + time_updated=2, + preview_url="", + consumer_app_id=550, + result=1, + ) + monkeypatch.setattr(steam_workshop, "fetch_metadata_batch", lambda steam_ids, *, mode: [meta]) + refresh_calls = [] + + def fake_refresh_all(metas, cache_root, *, executor_workers=8, should_cancel=None): + refresh_calls.append((list(metas), executor_workers, should_cancel is not None)) + return steam_workshop.RefreshReport(downloaded=1, errors=0) + + monkeypatch.setattr(steam_workshop, "refresh_all", fake_refresh_all) + + with app.app_context(): + affected = job_worker._run_refresh_workshop_items( + on_stdout=lambda _line: None, + on_stderr=lambda _line: None, + should_cancel=lambda: False, + ) + + assert affected == [ids.overlay] + assert refresh_calls == [([meta], 1, True)] + + +def test_refresh_job_enqueues_build_overlay_without_locking_its_final_log( + overlay_seeded_worker, monkeypatch, tmp_path +) -> None: + app, ids = overlay_seeded_worker + monkeypatch.setenv("LEFT4ME_ROOT", str(tmp_path)) + + from l4d2web.services import steam_workshop + + with session_scope() as s: + wi = WorkshopItem( + steam_id="1001", + title="A", + filename="old.vpk", + file_url="https://example.com/old.vpk", + file_size=1, + time_updated=1, + ) + s.add(wi) + s.flush() + s.add(OverlayWorkshopItem(overlay_id=ids.overlay, workshop_item_id=wi.id)) + + meta = steam_workshop.WorkshopMetadata( + steam_id="1001", + title="A", + filename="new.vpk", + file_url="https://example.com/new.vpk", + file_size=2, + time_updated=2, + preview_url="", + consumer_app_id=550, + result=1, + ) + monkeypatch.setattr(steam_workshop, "fetch_metadata_batch", lambda steam_ids, *, mode: [meta]) + monkeypatch.setattr( + steam_workshop, + "refresh_all", + lambda metas, cache_root, **kwargs: steam_workshop.RefreshReport(downloaded=1, errors=0), + ) + + job_id = add_job(ids.user, "refresh_workshop_items", server_id=None) + + with app.app_context(): + assert run_worker_once() is True + + with session_scope() as s: + job = s.scalar(select(Job).where(Job.id == job_id)) + build_job = s.scalar( + select(Job).where( + Job.operation == "build_overlay", + Job.overlay_id == ids.overlay, + Job.state == "queued", + ) + ) + lines = [row.line for row in job_logs_for(s, job_id)] + + assert job is not None + assert job.state == "succeeded" + assert build_job is not None + assert "enqueued build_overlay for 1 overlay(s)" in lines