92d6ebbe82
Adds two managed system overlays (l4d2center-maps, cedapug-maps) that fetch curated map archives from upstream sources and reconcile addons symlinks for non-Steam maps. A daily systemd timer enqueues a coalesced refresh_global_overlays worker job; downloads, extraction, and rebuilds run in the existing job worker and surface in the job log UI. Schema: GlobalOverlaySource / GlobalOverlayItem / GlobalOverlayItemFile plus nullable Job.user_id so system jobs render as "system" in the UI. The new builder reconciles symlinks against the per-source vpk cache and leaves foreign symlinks untouched. Initialize-time guard refuses to mount a partial overlay if any expected vpk is missing from cache. Refresh service uses shutil.move to handle EXDEV when /tmp and the cache live on different filesystems. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
104 lines
3.2 KiB
Python
104 lines
3.2 KiB
Python
from __future__ import annotations
|
|
|
|
import csv
|
|
from dataclasses import dataclass
|
|
import hashlib
|
|
import html as html_lib
|
|
import io
|
|
import json
|
|
from urllib.parse import urljoin, urlparse
|
|
import re
|
|
|
|
import requests
|
|
|
|
|
|
REQUEST_TIMEOUT_SECONDS = 30
|
|
L4D2CENTER_CSV_URL = "https://l4d2center.com/maps/servers/index.csv"
|
|
CEDAPUG_CUSTOM_URL = "https://cedapug.com/custom"
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class GlobalMapManifestItem:
|
|
item_key: str
|
|
display_name: str
|
|
download_url: str
|
|
expected_vpk_name: str = ""
|
|
expected_size: int | None = None
|
|
expected_md5: str = ""
|
|
|
|
|
|
def fetch_l4d2center_manifest() -> tuple[str, list[GlobalMapManifestItem]]:
|
|
response = requests.get(L4D2CENTER_CSV_URL, timeout=REQUEST_TIMEOUT_SECONDS)
|
|
response.raise_for_status()
|
|
text = response.text
|
|
return _sha256(text), parse_l4d2center_csv(text)
|
|
|
|
|
|
def fetch_cedapug_manifest() -> tuple[str, list[GlobalMapManifestItem]]:
|
|
response = requests.get(CEDAPUG_CUSTOM_URL, timeout=REQUEST_TIMEOUT_SECONDS)
|
|
response.raise_for_status()
|
|
text = response.text
|
|
return _sha256(text), parse_cedapug_custom_html(text)
|
|
|
|
|
|
def parse_l4d2center_csv(raw: str) -> list[GlobalMapManifestItem]:
|
|
reader = csv.DictReader(io.StringIO(raw), delimiter=";")
|
|
expected = ["Name", "Size", "md5", "Download link"]
|
|
if reader.fieldnames != expected:
|
|
raise ValueError("expected L4D2Center CSV header: Name;Size;md5;Download link")
|
|
items: list[GlobalMapManifestItem] = []
|
|
for row in reader:
|
|
name = (row.get("Name") or "").strip()
|
|
size_raw = (row.get("Size") or "").strip()
|
|
md5 = (row.get("md5") or "").strip().lower()
|
|
url = (row.get("Download link") or "").strip()
|
|
if not name or not url:
|
|
continue
|
|
items.append(
|
|
GlobalMapManifestItem(
|
|
item_key=name,
|
|
display_name=name,
|
|
download_url=url,
|
|
expected_vpk_name=name,
|
|
expected_size=int(size_raw) if size_raw else None,
|
|
expected_md5=md5,
|
|
)
|
|
)
|
|
return items
|
|
|
|
|
|
def parse_cedapug_custom_html(raw: str) -> list[GlobalMapManifestItem]:
|
|
match = re.search(r"renderCustomMapDownloads\((\[.*?\])\)</script>", raw, re.DOTALL)
|
|
if match is None:
|
|
raise ValueError("CEDAPUG page did not contain renderCustomMapDownloads data")
|
|
rows = json.loads(match.group(1))
|
|
items: list[GlobalMapManifestItem] = []
|
|
for row in rows:
|
|
if len(row) < 3:
|
|
continue
|
|
label = str(row[1])
|
|
link = str(row[2])
|
|
if link.startswith("http"):
|
|
continue
|
|
if not link:
|
|
continue
|
|
url = urljoin(CEDAPUG_CUSTOM_URL, link)
|
|
parsed = urlparse(url)
|
|
basename = parsed.path.rsplit("/", 1)[-1]
|
|
items.append(
|
|
GlobalMapManifestItem(
|
|
item_key=basename,
|
|
display_name=_strip_html(label),
|
|
download_url=url,
|
|
)
|
|
)
|
|
return items
|
|
|
|
|
|
def _strip_html(raw: str) -> str:
|
|
no_tags = re.sub(r"<[^>]+>", "", raw)
|
|
return html_lib.unescape(no_tags).strip()
|
|
|
|
|
|
def _sha256(raw: str) -> str:
|
|
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|