Merge pull request #9 from alliedmodders/psy/amxx-asset-match

Add asset_match_filter option

Author: psychonic

README.md

@@ -72,8 +72,29 @@ log:
   level: "INFO"   # env: LOG__LEVEL
 ```
 
-`storage` is required. `database` and `repo` are optional — omitting them disables
-DB upsert and reconciliation.
+`storage` is optional. Omitting it disables PDB symbol store processing and build-drop
+downloads — useful for products that only need commit log tracking.
+`database` and `repo` are optional — omitting them disables DB upsert and reconciliation.
+
+### Repo Settings
+
+The `repo` section supports additional options:
+
+```yaml
+repo:
+  owner: "alliedmodders"
+  name: "sourcemod"
+  product_name: "sourcemod"
+  workflow_path: ".github/workflows/build-release.yml"  # Workflow to monitor
+  version_branches:                    # Maps version prefix → branch name
+    "1.12": "1.12-dev"
+    "1.13": "master"
+  reconcile_max_age_days: 90           # How far back reconciliation looks (null = no limit)
+  asset_match_filter: null             # If set, only release assets containing this string
+                                       # are matched for the windows/linux URL columns.
+                                       # Useful for multi-package releases, e.g. set to "base"
+                                       # to match only the base package archive.
+```
 
 ## Running the Server
 

app.py

@@ -78,6 +78,11 @@ class RepoSettings(BaseModel):
     # Prevents stale broken releases from blocking pagination indefinitely.
     # Set to null to disable.
     reconcile_max_age_days: int | None = 90
+    # When set, only release assets whose name contains this string are
+    # matched for the windows_url / linux_url columns.  Useful for
+    # multi-package releases (e.g. set to "base" to match only the base
+    # package archive).
+    asset_match_filter: str | None = None
 
     @property
     def full_name(self) -> str:
@@ -106,14 +111,14 @@ def settings_customise_sources(
         return (init_settings, env_settings, YamlConfigSettingsSource(settings_cls))
 
     api: ApiSettings = Field(default_factory=ApiSettings)
-    storage: StorageSettings
+    storage: StorageSettings | None = None
     github: GithubSettings = Field(default_factory=GithubSettings)
     database: DatabaseSettings | None = None
     repo: RepoSettings | None = None
     log: LogSettings = Field(default_factory=LogSettings)
 
 
-config = AppConfig()  # ty: ignore[missing-argument]
+config = AppConfig()
 
 logging.basicConfig(
     level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
@@ -141,12 +146,17 @@ def _run_reconcile():
             _releases_client,
             config.database,
             config.repo.version_branches,
-            drop_base_path=config.storage.build_drop_base_path,
-            process_symbols_fn=_process_pdb_artifact_for_sha,
+            drop_base_path=config.storage.build_drop_base_path
+            if config.storage
+            else None,
+            process_symbols_fn=_process_pdb_artifact_for_sha
+            if config.storage
+            else None,
             download_fn=lambda url, path: download_file(url, path),
             product_name=config.repo.product_name,
             max_age_days=config.repo.reconcile_max_age_days,
             commit_log_table=config.database.commit_log_table,
+            asset_match_filter=config.repo.asset_match_filter,
         )
     except Exception:
         logger.exception("Scheduled reconciliation failed")
@@ -236,6 +246,7 @@ def _process_symbols_only(
     auth_headers: dict,
 ) -> None:
     """Download PDB zip, extract, and commit to symstore. Serialized via _storage_lock."""
+    assert config.storage is not None
     base_symbols_path = config.storage.symbol_store_base_path
 
     product_symbols_path = Path(base_symbols_path).resolve()
@@ -320,7 +331,7 @@ def _process_pdb_artifact_for_sha(sha: str, product_name: str) -> None:
     download the PDB artifact, and process into symstore.
     Artifacts expire after ~90 days; logs a warning and returns if unavailable.
     """
-    if not _releases_client or not config.repo:
+    if not _releases_client or not config.repo or not config.storage:
         return
 
     run = _releases_client.find_workflow_run_for_commit(sha, config.repo.workflow_path)
@@ -378,13 +389,13 @@ def process_artifacts(
         auth_headers = {"Authorization": f"Bearer {token}"} if token else {}
 
         # Download and process symbol files
-        if symbols_url:
+        if symbols_url and config.storage:
             _process_symbols_only(symbols_url, safe_version, product_name, auth_headers)
 
         # Fetch the GitHub Release once; used for both DB upsert and build drop.
         release = None
         needs_release = (config.database and config.repo) or (
-            config.storage.build_drop_base_path and config.repo
+            config.storage and config.storage.build_drop_base_path and config.repo
         )
         if needs_release and _releases_client:
             try:
@@ -405,14 +416,15 @@ def process_artifacts(
                     config.database,
                     config.repo.version_branches,
                     commit_log_table=config.database.commit_log_table,
+                    asset_match_filter=config.repo.asset_match_filter,
                 )
             except Exception:
                 logger.exception(
                     "DB upsert failed for build %s (non-fatal)", build_version
                 )
 
         # Download build archives to the local drop directory as a backup mirror.
-        if release and config.storage.build_drop_base_path:
+        if release and config.storage and config.storage.build_drop_base_path:
             try:
                 version_prefix = ".".join(build_version.split(".")[:2])
                 drop_dir = Path(config.storage.build_drop_base_path) / version_prefix

reconciler.py

@@ -18,6 +18,7 @@ def upsert_from_release(
     db_config,
     version_branches: dict[str, str],
     commit_log_table: str = "sm_commit_log",
+    asset_match_filter: str | None = None,
 ) -> bool:
     """
     Resolve and upsert a single GitHub release to the DB.
@@ -48,7 +49,9 @@ def upsert_from_release(
         return False
 
     timestamp = client.release_timestamp(release)
-    windows_url, linux_url = client.parse_release_assets(release)
+    windows_url, linux_url = client.parse_release_assets(
+        release, asset_filter=asset_match_filter
+    )
 
     with get_connection(db_config) as conn:
         upsert_build(
@@ -121,6 +124,7 @@ def reconcile(
     product_name: str | None = None,
     max_age_days: int | None = 90,
     commit_log_table: str = "sm_commit_log",
+    asset_match_filter: str | None = None,
 ) -> int:
     """
     Fetch GitHub releases and reconcile DB records, build archives, and symbols.
@@ -193,7 +197,9 @@ def reconcile(
             all_done = False
 
             if needs_url_update:
-                windows_url, linux_url = client.parse_release_assets(release)
+                windows_url, linux_url = client.parse_release_assets(
+                    release, asset_filter=asset_match_filter
+                )
                 if windows_url or linux_url:
                     with get_connection(db_config) as conn:
                         update_build_urls(
@@ -226,7 +232,9 @@ def reconcile(
                     continue
 
                 timestamp = client.release_timestamp(release)
-                windows_url, linux_url = client.parse_release_assets(release)
+                windows_url, linux_url = client.parse_release_assets(
+                    release, asset_filter=asset_match_filter
+                )
 
                 with get_connection(db_config) as conn:
                     upsert_build(

releases.py

@@ -122,11 +122,20 @@ def parse_tag(tag: str) -> tuple[str, int] | None:
         return m.group(1), int(m.group(2))
 
     @staticmethod
-    def parse_release_assets(release: dict) -> tuple[str | None, str | None]:
-        """Extract (windows_url, linux_url) from release assets."""
+    def parse_release_assets(
+        release: dict, asset_filter: str | None = None
+    ) -> tuple[str | None, str | None]:
+        """Extract (windows_url, linux_url) from release assets.
+
+        If asset_filter is set, only assets whose name contains the filter
+        string are considered (e.g. "base" to match only the base package
+        in a multi-package release).
+        """
         windows_url = linux_url = None
         for asset in release.get("assets", []):
             name = asset["name"]
+            if asset_filter and asset_filter not in name:
+                continue
             if re.search(r"-windows\.zip$", name, re.IGNORECASE):
                 windows_url = asset["browser_download_url"]
             elif re.search(r"-linux\.tar\.gz$", name, re.IGNORECASE):