diff --git a/README.md b/README.md index 1e7fb5a..76e3a0a 100644 --- a/README.md +++ b/README.md @@ -72,8 +72,29 @@ log: level: "INFO" # env: LOG__LEVEL ``` -`storage` is required. `database` and `repo` are optional — omitting them disables -DB upsert and reconciliation. +`storage` is optional. Omitting it disables PDB symbol store processing and build-drop +downloads — useful for products that only need commit log tracking. +`database` and `repo` are optional — omitting them disables DB upsert and reconciliation. + +### Repo Settings + +The `repo` section supports additional options: + +```yaml +repo: + owner: "alliedmodders" + name: "sourcemod" + product_name: "sourcemod" + workflow_path: ".github/workflows/build-release.yml" # Workflow to monitor + version_branches: # Maps version prefix → branch name + "1.12": "1.12-dev" + "1.13": "master" + reconcile_max_age_days: 90 # How far back reconciliation looks (null = no limit) + asset_match_filter: null # If set, only release assets containing this string + # are matched for the windows/linux URL columns. + # Useful for multi-package releases, e.g. set to "base" + # to match only the base package archive. +``` ## Running the Server diff --git a/app.py b/app.py index a3c2d31..2f95061 100644 --- a/app.py +++ b/app.py @@ -78,6 +78,11 @@ class RepoSettings(BaseModel): # Prevents stale broken releases from blocking pagination indefinitely. # Set to null to disable. reconcile_max_age_days: int | None = 90 + # When set, only release assets whose name contains this string are + # matched for the windows_url / linux_url columns. Useful for + # multi-package releases (e.g. set to "base" to match only the base + # package archive). + asset_match_filter: str | None = None @property def full_name(self) -> str: @@ -106,14 +111,14 @@ def settings_customise_sources( return (init_settings, env_settings, YamlConfigSettingsSource(settings_cls)) api: ApiSettings = Field(default_factory=ApiSettings) - storage: StorageSettings + storage: StorageSettings | None = None github: GithubSettings = Field(default_factory=GithubSettings) database: DatabaseSettings | None = None repo: RepoSettings | None = None log: LogSettings = Field(default_factory=LogSettings) -config = AppConfig() # ty: ignore[missing-argument] +config = AppConfig() logging.basicConfig( level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" @@ -141,12 +146,17 @@ def _run_reconcile(): _releases_client, config.database, config.repo.version_branches, - drop_base_path=config.storage.build_drop_base_path, - process_symbols_fn=_process_pdb_artifact_for_sha, + drop_base_path=config.storage.build_drop_base_path + if config.storage + else None, + process_symbols_fn=_process_pdb_artifact_for_sha + if config.storage + else None, download_fn=lambda url, path: download_file(url, path), product_name=config.repo.product_name, max_age_days=config.repo.reconcile_max_age_days, commit_log_table=config.database.commit_log_table, + asset_match_filter=config.repo.asset_match_filter, ) except Exception: logger.exception("Scheduled reconciliation failed") @@ -236,6 +246,7 @@ def _process_symbols_only( auth_headers: dict, ) -> None: """Download PDB zip, extract, and commit to symstore. Serialized via _storage_lock.""" + assert config.storage is not None base_symbols_path = config.storage.symbol_store_base_path product_symbols_path = Path(base_symbols_path).resolve() @@ -320,7 +331,7 @@ def _process_pdb_artifact_for_sha(sha: str, product_name: str) -> None: download the PDB artifact, and process into symstore. Artifacts expire after ~90 days; logs a warning and returns if unavailable. """ - if not _releases_client or not config.repo: + if not _releases_client or not config.repo or not config.storage: return run = _releases_client.find_workflow_run_for_commit(sha, config.repo.workflow_path) @@ -378,13 +389,13 @@ def process_artifacts( auth_headers = {"Authorization": f"Bearer {token}"} if token else {} # Download and process symbol files - if symbols_url: + if symbols_url and config.storage: _process_symbols_only(symbols_url, safe_version, product_name, auth_headers) # Fetch the GitHub Release once; used for both DB upsert and build drop. release = None needs_release = (config.database and config.repo) or ( - config.storage.build_drop_base_path and config.repo + config.storage and config.storage.build_drop_base_path and config.repo ) if needs_release and _releases_client: try: @@ -405,6 +416,7 @@ def process_artifacts( config.database, config.repo.version_branches, commit_log_table=config.database.commit_log_table, + asset_match_filter=config.repo.asset_match_filter, ) except Exception: logger.exception( @@ -412,7 +424,7 @@ def process_artifacts( ) # Download build archives to the local drop directory as a backup mirror. - if release and config.storage.build_drop_base_path: + if release and config.storage and config.storage.build_drop_base_path: try: version_prefix = ".".join(build_version.split(".")[:2]) drop_dir = Path(config.storage.build_drop_base_path) / version_prefix diff --git a/reconciler.py b/reconciler.py index bbaf788..3f29d6b 100644 --- a/reconciler.py +++ b/reconciler.py @@ -18,6 +18,7 @@ def upsert_from_release( db_config, version_branches: dict[str, str], commit_log_table: str = "sm_commit_log", + asset_match_filter: str | None = None, ) -> bool: """ Resolve and upsert a single GitHub release to the DB. @@ -48,7 +49,9 @@ def upsert_from_release( return False timestamp = client.release_timestamp(release) - windows_url, linux_url = client.parse_release_assets(release) + windows_url, linux_url = client.parse_release_assets( + release, asset_filter=asset_match_filter + ) with get_connection(db_config) as conn: upsert_build( @@ -121,6 +124,7 @@ def reconcile( product_name: str | None = None, max_age_days: int | None = 90, commit_log_table: str = "sm_commit_log", + asset_match_filter: str | None = None, ) -> int: """ Fetch GitHub releases and reconcile DB records, build archives, and symbols. @@ -193,7 +197,9 @@ def reconcile( all_done = False if needs_url_update: - windows_url, linux_url = client.parse_release_assets(release) + windows_url, linux_url = client.parse_release_assets( + release, asset_filter=asset_match_filter + ) if windows_url or linux_url: with get_connection(db_config) as conn: update_build_urls( @@ -226,7 +232,9 @@ def reconcile( continue timestamp = client.release_timestamp(release) - windows_url, linux_url = client.parse_release_assets(release) + windows_url, linux_url = client.parse_release_assets( + release, asset_filter=asset_match_filter + ) with get_connection(db_config) as conn: upsert_build( diff --git a/releases.py b/releases.py index 4824f97..1a31665 100644 --- a/releases.py +++ b/releases.py @@ -122,11 +122,20 @@ def parse_tag(tag: str) -> tuple[str, int] | None: return m.group(1), int(m.group(2)) @staticmethod - def parse_release_assets(release: dict) -> tuple[str | None, str | None]: - """Extract (windows_url, linux_url) from release assets.""" + def parse_release_assets( + release: dict, asset_filter: str | None = None + ) -> tuple[str | None, str | None]: + """Extract (windows_url, linux_url) from release assets. + + If asset_filter is set, only assets whose name contains the filter + string are considered (e.g. "base" to match only the base package + in a multi-package release). + """ windows_url = linux_url = None for asset in release.get("assets", []): name = asset["name"] + if asset_filter and asset_filter not in name: + continue if re.search(r"-windows\.zip$", name, re.IGNORECASE): windows_url = asset["browser_download_url"] elif re.search(r"-linux\.tar\.gz$", name, re.IGNORECASE):