fix(l4d2-web): keep workshop refresh responsive

Limit workshop refresh downloads to one worker and commit build-overlay enqueue work before writing the final job log so SQLite locks do not wedge the web process.

l4d2web/services/job_worker.py

@@ -28,6 +28,7 @@ ACTIVE_JOB_STATES = {"running", "cancelling"}
 SERVER_OPERATIONS = {"initialize", "start", "stop", "delete"}
 OVERLAY_OPERATIONS = {"build_overlay"}
 GLOBAL_OPERATIONS = {"install", "refresh_workshop_items"}
+WORKSHOP_REFRESH_DOWNLOAD_WORKERS = 1
 
 _claim_lock = threading.Lock()
 _log_lock = threading.RLock()
@@ -434,7 +435,10 @@ def _run_refresh_workshop_items(
     on_stdout(f"downloading {len(download_metas)} items")
     if download_metas:
         report = steam_workshop.refresh_all(
-            download_metas, workshop_cache_root(), should_cancel=should_cancel
+            download_metas,
+            workshop_cache_root(),
+            executor_workers=WORKSHOP_REFRESH_DOWNLOAD_WORKERS,
+            should_cancel=should_cancel,
         )
         on_stdout(
             f"download phase complete (downloaded={report.downloaded} errors={report.errors})"
@@ -479,9 +483,8 @@ def _run_refresh_workshop_items(
                 if user_id is None:
                     continue
             enqueue_build_overlay(db, overlay_id=ov_id, user_id=user_id)
-        on_stdout(
-            f"enqueued build_overlay for {len(affected_overlay_ids)} overlay(s)"
-        )
+
+    on_stdout(f"enqueued build_overlay for {len(affected_overlay_ids)} overlay(s)")
     return list(affected_overlay_ids)
 
 

l4d2web/tests/test_job_worker.py

@@ -587,3 +587,116 @@ def test_run_worker_once_dispatches_refresh(overlay_seeded_worker, monkeypatch,
     assert refresh_calls == [True]
     job = load_job(job_id)
     assert job.state == "succeeded"
+
+
+def test_refresh_downloads_serially_to_keep_web_worker_responsive(
+    overlay_seeded_worker, monkeypatch, tmp_path
+) -> None:
+    app, ids = overlay_seeded_worker
+    monkeypatch.setenv("LEFT4ME_ROOT", str(tmp_path))
+
+    from l4d2web.services import job_worker, steam_workshop
+
+    with session_scope() as s:
+        wi = WorkshopItem(
+            steam_id="1001",
+            title="A",
+            filename="old.vpk",
+            file_url="https://example.com/old.vpk",
+            file_size=1,
+            time_updated=1,
+        )
+        s.add(wi)
+        s.flush()
+        s.add(OverlayWorkshopItem(overlay_id=ids.overlay, workshop_item_id=wi.id))
+
+    meta = steam_workshop.WorkshopMetadata(
+        steam_id="1001",
+        title="A",
+        filename="new.vpk",
+        file_url="https://example.com/new.vpk",
+        file_size=2,
+        time_updated=2,
+        preview_url="",
+        consumer_app_id=550,
+        result=1,
+    )
+    monkeypatch.setattr(steam_workshop, "fetch_metadata_batch", lambda steam_ids, *, mode: [meta])
+    refresh_calls = []
+
+    def fake_refresh_all(metas, cache_root, *, executor_workers=8, should_cancel=None):
+        refresh_calls.append((list(metas), executor_workers, should_cancel is not None))
+        return steam_workshop.RefreshReport(downloaded=1, errors=0)
+
+    monkeypatch.setattr(steam_workshop, "refresh_all", fake_refresh_all)
+
+    with app.app_context():
+        affected = job_worker._run_refresh_workshop_items(
+            on_stdout=lambda _line: None,
+            on_stderr=lambda _line: None,
+            should_cancel=lambda: False,
+        )
+
+    assert affected == [ids.overlay]
+    assert refresh_calls == [([meta], 1, True)]
+
+
+def test_refresh_job_enqueues_build_overlay_without_locking_its_final_log(
+    overlay_seeded_worker, monkeypatch, tmp_path
+) -> None:
+    app, ids = overlay_seeded_worker
+    monkeypatch.setenv("LEFT4ME_ROOT", str(tmp_path))
+
+    from l4d2web.services import steam_workshop
+
+    with session_scope() as s:
+        wi = WorkshopItem(
+            steam_id="1001",
+            title="A",
+            filename="old.vpk",
+            file_url="https://example.com/old.vpk",
+            file_size=1,
+            time_updated=1,
+        )
+        s.add(wi)
+        s.flush()
+        s.add(OverlayWorkshopItem(overlay_id=ids.overlay, workshop_item_id=wi.id))
+
+    meta = steam_workshop.WorkshopMetadata(
+        steam_id="1001",
+        title="A",
+        filename="new.vpk",
+        file_url="https://example.com/new.vpk",
+        file_size=2,
+        time_updated=2,
+        preview_url="",
+        consumer_app_id=550,
+        result=1,
+    )
+    monkeypatch.setattr(steam_workshop, "fetch_metadata_batch", lambda steam_ids, *, mode: [meta])
+    monkeypatch.setattr(
+        steam_workshop,
+        "refresh_all",
+        lambda metas, cache_root, **kwargs: steam_workshop.RefreshReport(downloaded=1, errors=0),
+    )
+
+    job_id = add_job(ids.user, "refresh_workshop_items", server_id=None)
+
+    with app.app_context():
+        assert run_worker_once() is True
+
+    with session_scope() as s:
+        job = s.scalar(select(Job).where(Job.id == job_id))
+        build_job = s.scalar(
+            select(Job).where(
+                Job.operation == "build_overlay",
+                Job.overlay_id == ids.overlay,
+                Job.state == "queued",
+            )
+        )
+        lines = [row.line for row in job_logs_for(s, job_id)]
+
+    assert job is not None
+    assert job.state == "succeeded"
+    assert build_job is not None
+    assert "enqueued build_overlay for 1 overlay(s)" in lines