diff --git a/.github/path-filters.yml b/.github/path-filters.yml index 916fec0cecb..a845b4b86e1 100644 --- a/.github/path-filters.yml +++ b/.github/path-filters.yml @@ -1,3 +1,7 @@ +# This file is generated by scripts/update-gha-workflows.ts +# +# To regenerate: npm run refresh-gh-workflow + # The `&global` anchor defines a set of common paths to include by reference in the other filters. global: &global - ".github/path-filters.yml" @@ -14,9 +18,6 @@ any-workspace: task-herder: - *global - "packages/task-herder/**" -scratch-media-lib-scripts: - - *global - - "packages/scratch-media-lib-scripts/**" scratch-svg-renderer: - *global - "packages/scratch-svg-renderer/**" @@ -35,3 +36,6 @@ scratch-gui: - "packages/scratch-render/**" - "packages/scratch-svg-renderer/**" - "packages/scratch-vm/**" +scratch-media-lib-scripts: + - *global + - "packages/scratch-media-lib-scripts/**" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 08c1ab8267c..e4e0977601f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -192,11 +192,7 @@ jobs: run: npm publish --access=public --tag="$NPM_TAG" --ignore-scripts --workspace=@scratch/task-herder - name: Publish scratch-media-lib-scripts - run: | - npm run build --workspace @scratch/scratch-media-lib-scripts - npm publish --access=public --tag="$NPM_TAG" --workspace=@scratch/scratch-media-lib-scripts - env: - NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} + run: npm publish --access=public --tag="$NPM_TAG" --ignore-scripts --workspace=@scratch/scratch-media-lib-scripts - name: Push to Develop shell: bash diff --git a/.gitignore b/.gitignore index 780f5bfd972..813a6469d83 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,9 @@ /monorepo.tmp /monorepo.cache +# add-repo.sh +/add-repo.tmp + # Logs logs *.log diff --git a/package.json b/package.json index 3dfacb9b7d0..bbbd41480df 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "build-monorepo": "cross-env-shell ./scripts/build-monorepo.sh", "clean": "npm run --workspaces clean", "prepare": "husky install", - "refresh-gh-workflow": "ts-node scripts/build-gha-workflows.ts", + "refresh-gh-workflow": "ts-node scripts/update-gha-workflows.ts", "test": "npm test --workspaces", "update-legal": "npm --workspaces exec -c 'rm -f ./{LICENSE,TRADEMARK} && cp -f ../../{LICENSE,TRADEMARK} .'", "version": "cross-env-shell ./scripts/npm-version.sh" diff --git a/scripts/add-repo.sh b/scripts/add-repo.sh new file mode 100755 index 00000000000..f2d5198c90f --- /dev/null +++ b/scripts/add-repo.sh @@ -0,0 +1,467 @@ +#!/bin/bash + +# add-repo.sh — Add an existing GitHub repository into the scratch-editor monorepo. +# +# This script imports a single repo with full git history into the current monorepo +# checkout. It rewrites the source repo's history so all files live under +# packages//, merges it into the current branch, rewires inter-package +# dependencies, updates the root workspaces list, and regenerates CI workflows. +# +# Prerequisites: +# - git-filter-repo (brew install git-filter-repo) +# - sponge (brew install moreutils) +# - jq (brew install jq) +# +# Usage: +# ./scripts/add-repo.sh [options] +# +# Options: +# --source-branch Branch to import from the source repo (default: auto-detect) +# --org GitHub organization (default: scratchfoundation) +# --cache-dir Local cache dir containing a clone of the repo (default: ./..) +# --no-ci Skip CI workflow regeneration +# --help Show this help message +# +# Examples: +# ./scripts/add-repo.sh scratch-audio +# ./scripts/add-repo.sh scratch-storage --source-branch develop +# ./scripts/add-repo.sh scratch-paint --org myfork --cache-dir ~/GitHub + +set -e + +### Defaults ### + +GITHUB_ORG="scratchfoundation" +MONOREPO_URL="https://github.com/scratchfoundation/scratch-editor.git" +NPM_ORGANIZATION="@scratch" +BUILD_CACHE="./.." +BUILD_TMP="./add-repo.tmp" +SOURCE_BRANCH="develop" +SKIP_CI=false + +### Argument parsing ### + +usage() { + sed -n '/^# Usage:/,/^[^#]/{ /^#/s/^# \{0,1\}//p; }' "$0" + exit "${1:-0}" +} + +REPO_NAME="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --source-branch) + SOURCE_BRANCH="$2" + shift 2 + ;; + --org) + GITHUB_ORG="$2" + shift 2 + ;; + --cache-dir) + BUILD_CACHE="$2" + shift 2 + ;; + --no-ci) + SKIP_CI=true + shift + ;; + --help|-h) + usage 0 + ;; + -*) + echo "Unknown option: $1" >&2 + usage 1 + ;; + *) + if [ -z "$REPO_NAME" ]; then + REPO_NAME="$1" + else + echo "Unexpected argument: $1" >&2 + usage 1 + fi + shift + ;; + esac +done + +if [ -z "$REPO_NAME" ]; then + echo "Error: repository name is required." >&2 + usage 1 +fi + +MONOREPO_ROOT="$(git rev-parse --show-toplevel)" +PACKAGE_DIR="packages/${REPO_NAME}" +PACKAGE_PATH="${MONOREPO_ROOT}/${PACKAGE_DIR}" + +### Prerequisite checks ### + +echo "==> Checking prerequisites..." + +if ! git filter-repo -h &> /dev/null; then + echo "Error: Please install git-filter-repo. You can try with:" >&2 + echo "- brew install git-filter-repo" >&2 + echo "- sudo apt install git-filter-repo" >&2 + exit 1 +fi + +if ! command -v sponge &> /dev/null; then + echo "Error: Please install the 'sponge' command. You can try with:" >&2 + echo "- brew install moreutils" >&2 + echo "- sudo apt install moreutils" >&2 + exit 1 +fi + +if ! command -v jq &> /dev/null; then + echo "Error: Please install jq. You can try with:" >&2 + echo "- brew install jq" >&2 + echo "- sudo apt install jq" >&2 + exit 1 +fi + +if [ -d "$PACKAGE_PATH" ]; then + echo "Error: ${PACKAGE_DIR} already exists in the monorepo." >&2 + echo "If you want to re-add it, remove it first." >&2 + exit 1 +fi + +if [ -d "$BUILD_TMP" ]; then + echo "Error: Temporary directory ${BUILD_TMP} already exists." >&2 + echo "A previous run may have failed. Remove it with: rm -rf ${BUILD_TMP}" >&2 + exit 1 +fi + +CURRENT_BRANCH="$(git rev-parse --abbrev-ref HEAD)" +echo " Monorepo root: ${MONOREPO_ROOT}" +echo " Current branch: ${CURRENT_BRANCH}" +echo " Target package: ${PACKAGE_DIR}" +echo " GitHub org: ${GITHUB_ORG}" +echo " Source repo: ${GITHUB_ORG}/${REPO_NAME}" + +### Helper functions (adapted from build-monorepo.sh) ### + +# Thanks to https://stackoverflow.com/a/17841619 +join_args() { + local d=${1-} f=${2-} + if shift 2; then + printf %s "$f" "${@/#/$d}" + fi +} + +# Clone a repository into BUILD_TMP as a bare repo +# Uses a local cache directory for speed if available +clone_repository() { + local repo="$1" + local org_and_repo="${GITHUB_ORG}/${repo}" + + echo "==> Cloning ${org_and_repo}..." + + mkdir -p "$BUILD_TMP" + + if [ -d "${BUILD_CACHE}/${repo}" ]; then + echo " Using local cache at ${BUILD_CACHE}/${repo}" + git -C "${BUILD_CACHE}/${repo}" fetch --all + git -C "$BUILD_TMP" clone --bare --dissociate \ + --reference "$(realpath "$BUILD_CACHE")/${repo}" \ + "git@github.com:${org_and_repo}.git" "${repo}" + else + echo " No local cache found, cloning directly from GitHub..." + git -C "$BUILD_TMP" clone --bare "git@github.com:${org_and_repo}.git" "${repo}" + fi + + # Disconnect from the reference repo + git -C "${BUILD_TMP}/${repo}" repack -a + rm -f "${BUILD_TMP}/${repo}/.git/objects/info/alternates" +} + +# Rewrite all paths in the cloned repo to live under a subdirectory +# Handles repos with and without submodules +move_repository_subdirectory() { + local repo="$1" + local subdirectory="$2" + + echo "==> Rewriting history to move files under ${subdirectory}..." + + # Make filter-repo accept this as a fresh clone + git -C "${BUILD_TMP}/${repo}" gc + + local has_submodules + has_submodules=$( + git -C "${BUILD_TMP}/${repo}" branch --format="%(refname:short)" | while read branch; do + if git -C "${BUILD_TMP}/${repo}" cat-file -e "${branch}:.gitmodules" &> /dev/null; then + echo "yep" + break + fi + done + ) + + # rewrite history as if all this work happened in a subdirectory + # "git mv" is simpler but makes history much less visible + if [ "$has_submodules" != "yep" ]; then + echo " Repository does NOT have submodules" + git -C "${BUILD_TMP}/${repo}" filter-repo --to-subdirectory-filter "$subdirectory" + else + echo " Repository DOES have submodules" + # the .gitmodules file must stay in the repository root, but the paths inside it must be rewritten + # this is complicated for the reasons described here: https://github.com/newren/git-filter-repo/issues/158 + # this is also slower, so we only do it for repositories that have submodules + # if we have more than one, this will cause merge conflicts + git -C "${BUILD_TMP}/${repo}" filter-repo \ + --filename-callback "return filename if filename == b'.gitmodules' else b'${subdirectory}'+filename" \ + --blob-callback "if blob.data.startswith(b'[submodule '): blob.data = blob.data.replace(b'path = ', b'path = ${subdirectory}')" + fi +} + +# Get the list of existing monorepo workspace package names (without @scratch/ prefix) +# e.g. "scratch-gui scratch-vm scratch-render scratch-svg-renderer" +get_existing_packages() { + jq -r '.workspaces[]' "${MONOREPO_ROOT}/package.json" | sed 's|packages/||' +} + +# Report that replacing a dependency with the local monorepo version failed +# $1: the name of the repository +# $2: the branch that was being built +# $3: the dependency that failed to install +package_replacement_error () { + echo "***ERROR***" + echo "Could not replace a dependency with the local monorepo version." + echo "Failed to replace $3 in $1#$2" | tee -a "monorepo.errors.log" + #exit 1 # uncomment this to make it a fatal error + echo "Attempting to continue anyway..." +} + +### Step 1: Clone and rewrite history ### + +echo "" +echo "==========================================" +echo " Adding ${REPO_NAME} to the monorepo" +echo "==========================================" +echo "" + +clone_repository "$REPO_NAME" + +move_repository_subdirectory "$REPO_NAME" "${PACKAGE_DIR}/" + +### Step 2: Verify source branch ### + +if [ -z "$(git -C "${BUILD_TMP}/${REPO_NAME}" branch --list "$SOURCE_BRANCH")" ]; then + echo "Error: Branch '${SOURCE_BRANCH}' not found in ${REPO_NAME}." >&2 + echo "Available branches:" >&2 + git -C "${BUILD_TMP}/${REPO_NAME}" branch --list | sed 's/^/ /' >&2 + rm -rf "$BUILD_TMP" + exit 1 +fi +echo "==> Using specified source branch: ${SOURCE_BRANCH}" + +### Step 3: Merge into current branch ### + +echo "==> Merging ${REPO_NAME}#${SOURCE_BRANCH} into ${CURRENT_BRANCH}..." + +REMOTE_NAME="temp-${REPO_NAME}" +git remote add "$REMOTE_NAME" "$(realpath "${BUILD_TMP}")/${REPO_NAME}" +git fetch --no-tags "$REMOTE_NAME" + +MERGE_MESSAGE="feat: add ${REPO_NAME}#${SOURCE_BRANCH} as ${PACKAGE_DIR}" +git merge --no-ff --allow-unrelated-histories "${REMOTE_NAME}/${SOURCE_BRANCH}" -m "$MERGE_MESSAGE" + +git remote remove "$REMOTE_NAME" + +echo " Merge complete." + +# Clean up the temporary clone now — before any `git add -A` can accidentally stage it +echo "==> Cleaning up temporary clone..." +rm -rf "$BUILD_TMP" + +### Step 4: Fixup the new package ### + +echo "==> Fixing up ${PACKAGE_DIR}/package.json..." + +# remove repository-level configuration and dependencies, like Renovate and Husky +# do not remove configuration and dependencies that could vary between packages, like semantic-release +# do not remove content like .github/ that may be useful as reference when building the monorepo equivalent +# it would be nice to merge all the package-lock.json files into one but it's not clear how to do that +# just remove the package-lock.json files for now, and build a new one with "npm i" later +rm -rf "${PACKAGE_PATH}/.husky" \ + "${PACKAGE_PATH}/package-lock.json" \ + "${PACKAGE_PATH}/renovate.json" \ + "${PACKAGE_PATH}/renovate.json5" + +# Rewrite package.json: rename package, set version, set monorepo URL, strip Husky/commitlint config +MONOREPO_VERSION=$(jq -r '.version' "${MONOREPO_ROOT}/package.json") +if [ -r "${PACKAGE_PATH}/package.json" ]; then + jq -f --arg PACKAGE_NAME "${NPM_ORGANIZATION}/${REPO_NAME}" \ + --arg MONOREPO_URL "$MONOREPO_URL" \ + --arg MONOREPO_VERSION "$MONOREPO_VERSION" \ + <(join_args ' | ' \ + '.name |= $PACKAGE_NAME' \ + '.version |= $MONOREPO_VERSION' \ + '.repository.url |= $MONOREPO_URL' \ + 'if .scripts.prepare == "husky install" then del(.scripts.prepare) else . end' \ + 'if .scripts == {} then del(.scripts.prepare) else . end' \ + 'del(.config.commitizen)' \ + 'if .config == {} then del(.config) else . end' \ + 'del(.devDependencies."@commitlint/cli")' \ + 'del(.devDependencies."@commitlint/config-conventional")' \ + 'del(.devDependencies."@commitlint/travis-cli")' \ + 'del(.devDependencies."cz-conventional-changelog")' \ + 'del(.devDependencies."husky")' \ + 'if .devDependencies == {} then del(.devDependencies) else . end' \ + ) "${PACKAGE_PATH}/package.json" | sponge "${PACKAGE_PATH}/package.json" +fi + +### Step 5: Update root workspaces list ### + +echo "==> Updating root package.json workspaces..." + +# Add the new package to the workspaces array BEFORE rewiring dependencies, +# so npm can resolve it as a local workspace package rather than fetching from the registry. +# NOTE: This prepends it to the beginning so it is built before packages that depend on it. +WORKSPACE_ENTRY="packages/${REPO_NAME}" +if ! jq -e ".workspaces | index(\"${WORKSPACE_ENTRY}\")" "${MONOREPO_ROOT}/package.json" > /dev/null 2>&1; then + jq ".workspaces |= [\"${WORKSPACE_ENTRY}\"] + ." "${MONOREPO_ROOT}/package.json" \ + | sponge "${MONOREPO_ROOT}/package.json" + echo " Added '${WORKSPACE_ENTRY}' to workspaces." +else + echo " '${WORKSPACE_ENTRY}' already in workspaces." +fi + +### Step 6: Rewire inter-package dependencies ### + +echo "==> Rewiring inter-package dependencies..." + +# Collect all monorepo packages: existing ones + the newly added one +EXISTING_PACKAGES=$(get_existing_packages) +ALL_PACKAGES="${EXISTING_PACKAGES} ${REPO_NAME}" + +# For the newly added package, replace deps pointing to other monorepo packages +# Also check existing packages for deps pointing to the newly added repo +for PACKAGE in $ALL_PACKAGES; do + PACKAGE_JSON="${MONOREPO_ROOT}/packages/${PACKAGE}/package.json" + + if [ ! -r "$PACKAGE_JSON" ]; then + continue + fi + + DEPS="" + DEVDEPS="" + OPTDEPS="" + PEERDEPS="" + + for DEP in $ALL_PACKAGES; do + # Check each dependency type for references to other monorepo packages by their + # un-prefixed name (e.g. "scratch-vm" instead of "@scratch/scratch-vm") + if jq -e ".dependencies.\"${DEP}\"" "$PACKAGE_JSON" > /dev/null 2>&1; then + jq "del(.dependencies.\"${DEP}\")" "$PACKAGE_JSON" | sponge "$PACKAGE_JSON" + DEPS="$DEPS ${DEP}@*" + fi + if jq -e ".devDependencies.\"${DEP}\"" "$PACKAGE_JSON" > /dev/null 2>&1; then + jq "del(.devDependencies.\"${DEP}\")" "$PACKAGE_JSON" | sponge "$PACKAGE_JSON" + DEVDEPS="$DEVDEPS ${DEP}@*" + fi + if jq -e ".optionalDependencies.\"${DEP}\"" "$PACKAGE_JSON" > /dev/null 2>&1; then + jq "del(.optionalDependencies.\"${DEP}\")" "$PACKAGE_JSON" | sponge "$PACKAGE_JSON" + OPTDEPS="$OPTDEPS ${DEP}@*" + fi + if jq -e ".peerDependencies.\"${DEP}\"" "$PACKAGE_JSON" > /dev/null 2>&1; then + jq "del(.peerDependencies.\"${DEP}\")" "$PACKAGE_JSON" | sponge "$PACKAGE_JSON" + PEERDEPS="$PEERDEPS ${DEP}@*" + fi + done + + # Re-add as workspace dependencies with the @scratch/ prefix + for DEP in $DEPS; do + npm install --force --save --save-exact \ + "${NPM_ORGANIZATION}/${DEP}" -w "${NPM_ORGANIZATION}/${PACKAGE}" \ + || package_replacement_error "$PACKAGE" "$DEP" + done + for DEP in $DEVDEPS; do + npm install --force --save-dev --save-exact \ + "${NPM_ORGANIZATION}/${DEP}" -w "${NPM_ORGANIZATION}/${PACKAGE}" \ + || package_replacement_error "$PACKAGE" "$DEP" + done + for DEP in $OPTDEPS; do + npm install --force --save-optional --save-exact \ + "${NPM_ORGANIZATION}/${DEP}" -w "${NPM_ORGANIZATION}/${PACKAGE}" \ + || package_replacement_error "$PACKAGE" "$DEP" + done + for DEP in $PEERDEPS; do + npm install --force --save-peer --save-exact \ + "${NPM_ORGANIZATION}/${DEP}" -w "${NPM_ORGANIZATION}/${PACKAGE}" \ + || package_replacement_error "$PACKAGE" "$DEP" + done +done + +# Replace require/import references to the new repo's un-prefixed name across the codebase +echo "==> Updating require/import references for ${REPO_NAME}..." +find "${MONOREPO_ROOT}" -type f \ + -not -path '*/.git/*' \ + -not -path '*/node_modules/*' \ + -exec grep -Il "$REPO_NAME" {} \; 2>/dev/null | while read -r file; do + sed -i '' -e "s:\(require(\|from \|resolve(\|node_modules\)\(['\"/]\)${REPO_NAME}\(['\"/]\):\1\2${NPM_ORGANIZATION}/${REPO_NAME}\3:g" "$file" +done + +# Also update references in existing packages that point to the new repo +# (These are require/import statements in files that were already in the monorepo) + +### Step 7: Install dependencies ### + +echo "==> Running npm install..." + +npm install --prefer-offline --no-audit --no-fund +npm install --package-lock-only 2>/dev/null || true # normalize package-lock.json + +### Step 8: Commit fixup changes ### + +echo "==> Committing fixup changes..." + +git add -A +if ! git diff --cached --quiet; then + git commit -m "feat: integrate ${REPO_NAME} into monorepo + +- Renamed package to ${NPM_ORGANIZATION}/${REPO_NAME} +- Removed repo-level config (.husky, renovate, commitlint) +- Rewired inter-package dependencies to use workspace versions +- Added to root workspaces list +- Regenerated package-lock.json" +else + echo " No fixup changes to commit." +fi + +### Step 9: Regenerate CI workflows ### + +if [ "$SKIP_CI" = false ]; then + echo "==> Regenerating CI workflows..." + npm run refresh-gh-workflow + + git add -A + if ! git diff --cached --quiet; then + git commit -m "ci: regenerate workflows after adding ${REPO_NAME}" + else + echo " No CI workflow changes to commit." + fi +else + echo "==> Skipping CI workflow regeneration (--no-ci)." + echo " Run 'npm run refresh-gh-workflow' manually when ready." +fi + +### Step 10: Done ### + +echo "" +echo "==========================================" +echo " Successfully added ${REPO_NAME}!" +echo "==========================================" +echo "" +echo "Summary:" +echo " - Merged ${REPO_NAME}#${SOURCE_BRANCH} into ${CURRENT_BRANCH}" +echo " - Package location: ${PACKAGE_DIR}/" +echo " - Package name: ${NPM_ORGANIZATION}/${REPO_NAME}" +echo "" +echo "Recommended next steps:" +echo " 1. Review the merge commits: git log --oneline -10" +echo " 2. Verify the package: ls ${PACKAGE_DIR}/" +echo " 3. Check workspace resolution: npm ls ${NPM_ORGANIZATION}/${REPO_NAME}" +echo " 4. Review generated CI changes:" +echo " .github/path-filters.yml" +echo " .github/workflows/publish.yml" +echo " 5. Run tests: npm test -w ${NPM_ORGANIZATION}/${REPO_NAME}" +echo "" diff --git a/scripts/build-gha-workflows.ts b/scripts/build-gha-workflows.ts deleted file mode 100644 index 4567a46046c..00000000000 --- a/scripts/build-gha-workflows.ts +++ /dev/null @@ -1,258 +0,0 @@ -// Recalculates the path filters for the dynamic CircleCI configuration -// Usage: node .circleci/refresh-path-filters.mjs -// Then copy the output into the CircleCI config files -// See https://circleci.com/docs/using-dynamic-configuration/ -// See https://github.com/circle-makotom/circle-advanced-setup-workflow - -type WorkflowMeta = { - filename: string; - displayName: string; -} - -// BEGIN CONFIGURATION - -const pathsFilterAction = 'dorny/paths-filter@v2'; - -const mainWorkflowMeta: WorkflowMeta = { - filename: 'ci-cd.yml', - displayName: 'CI/CD', -}; - -function getWorkspaceWorkflowMeta(workspace: Workspace): WorkflowMeta { - return { - filename: `workspace-${workspace.yamlName}.yml`, - displayName: `${workspace.name} (placeholder)`, - }; -} - -// END CONFIGURATION - -import {exec} from 'child_process'; -import fs from 'fs'; -import {promisify} from 'util'; -import path from 'path'; - -const execAsync = promisify(exec); - -enum DependencyType { - Dependencies = 'dependencies', - DevDependencies = 'devDependencies', - PeerDependencies = 'peerDependencies' -} - -type PackageJson = { - name: string; - location: string; - dependencies: {[name: string]: string}; - devDependencies: {[name: string]: string}; - peerDependencies: {[name: string]: string}; -}; - -type Workspace = { - name: string; - location: string; - yamlName: string; - dependencies: string[]; - devDependencies: string[]; - peerDependencies: string[]; - deepDependencies: string[]; -}; - -type WorkspaceMap = {[name: string]: Workspace} -type WorkspaceList = Workspace[]; - -const workspaceTemplate = fs.readFileSync(path.join(__dirname, 'workspace-template.yml'), 'utf8'); - -/** - * Calculate dependencies between workspaces in this repository. - * @param workspaces The result of `npm query .workspace`. - * @param depTypes The dependency types to include in the calculation. - * @returns A map of workspace names to their dependencies. - */ -function calculateDependencies(workspaces: Array, depTypes: DependencyType[]): WorkspaceMap { - const workspaceNames = workspaces.map(workspace => workspace.name); - const dependencies = workspaces.reduce((bag, workspace) => { - const workspaceEntry = depTypes.reduce((workspaceDeps, depType) => { - const deps = workspace[depType]; - workspaceDeps[depType] = deps ? Object.keys(deps).filter(dep => workspaceNames.includes(dep)) : []; - return workspaceDeps; - }, ({name: workspace.name})); - workspaceEntry.location = workspace.location; - workspaceEntry.yamlName = path.basename(workspace.location).replace(/@/g, '').replace(/[/.]/g, '-'); - bag[workspace.name] = workspaceEntry; - return bag; - }, ({})); - console.log('Calculating deep dependencies...'); - const addDeepDependencies = (deepDependencies: string[], workspaceName: string, depType: DependencyType) => { - const depDeps = dependencies[workspaceName][depType] || []; - for (let dep of depDeps) { - if (deepDependencies.includes(dep)) { - continue; - } - deepDependencies.push(dep); - addDeepDependencies(deepDependencies, dep, depType); - } - }; - for (let workspace of workspaces) { - const deepDependencies = dependencies[workspace.name].deepDependencies = [workspace.name]; - for (let depType of depTypes) { - addDeepDependencies(deepDependencies, workspace.name, depType); - } - } - return dependencies; -}; - -/** - * Sort workspaces in dependency order. - * @param workspaces The workspace map to sort. - * @returns The workspaces sorted in dependency order. - */ -function sortWorkspaces(workspaces: WorkspaceMap): WorkspaceList { - // TODO: is this reliable? Do we need a full topo sort? - const sortedWorkspaces = Object.values(workspaces).sort((a, b) => { - if (a.deepDependencies.includes(b.name)) { - return 1; - } - if (b.deepDependencies.includes(a.name)) { - return -1; - } - return 0; - }); - return sortedWorkspaces; -} - -async function generateWorkflow(sortedWorkspaces: WorkspaceList, workspaces: WorkspaceMap): Promise { - let workflowFileHandle: fs.promises.FileHandle | undefined; - let workflowStream: fs.WriteStream | undefined; - try { - workflowFileHandle = await fs.promises.open(path.join('.github', 'workflows', mainWorkflowMeta.filename), 'w'); - workflowStream = workflowFileHandle.createWriteStream(); - workflowStream.write([ - `name: ${mainWorkflowMeta.displayName}`, - '', - 'on:', - ' push:', - ' workflow_dispatch:', - '', - 'concurrency:', - ' group: "${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}"', - ' cancel-in-progress: true', - '', - 'jobs:', - ].join('\n') + '\n'); - - generateChangesJob(workflowStream, sortedWorkspaces, workspaces); - - generateCalls(workflowStream, sortedWorkspaces, workspaces); - } finally { - workflowStream?.end(); - workflowStream?.close(); - await workflowFileHandle?.close(); - } -} - -function generateChangesJob(workflowStream: fs.WriteStream, sortedWorkspaces: WorkspaceList, workspaces: WorkspaceMap) { - workflowStream.write([ - ' changes:', - ' name: Detect affected workspaces', - ' runs-on: ubuntu-latest', - ' outputs:', - ].join('\n') + '\n'); - for (let workspace of sortedWorkspaces) { - workflowStream.write(` ${workspace.yamlName}: \${{ steps.filter.outputs.${workspace.yamlName} }}\n`); - } - workflowStream.write([ - ' steps:', - ' - uses: actions/checkout@v4 # TODO: skip this for PRs', - ` - uses: ${pathsFilterAction}`, - ' id: filter', - ' with:', - ' filters: |', - ' any-workspace:', - ' - ".github/workflows/workspace-*.yml"', - ' - "workspaces/**"', - ].join('\n') + '\n'); - for (let workspace of sortedWorkspaces) { - workflowStream.write([ - ` ${workspace.yamlName}:`, - ` - ".github/workflows/workspace-${workspace.yamlName}.yml"`, - ].join('\n') + '\n'); - for (let dep of workspace.deepDependencies.sort()) { - workflowStream.write(` - "${workspaces[dep].location}/**"\n`); - } - } - workflowStream.write([ - " - if: ${{ steps.filter.outputs.any-workspace == 'true' }}", - ' uses: actions/setup-node@v3', - ' with:', - ' cache: npm', - ' node-version-file: .nvmrc', - " - if: ${{ steps.filter.outputs.any-workspace == 'true' }}", - ' uses: ./.github/actions/install-dependencies', - ].join('\n') + '\n'); -} - -function generateCalls(workflowStream: fs.WriteStream, sortedWorkspaces: WorkspaceList, workspaces: WorkspaceMap) { - for (let workspace of sortedWorkspaces) { - workflowStream.write([ - ` ${workspace.yamlName}:`, - ` uses: ./.github/workflows/workspace-${workspace.yamlName}.yml`, - // By default, this job will only run if the jobs it 'needs' have succeeded. - // Instead, run even if some of those are skipped, but not if they failed or if the workflow was cancelled. - ` if: \${{ !failure() && !cancelled() && needs.changes.outputs.${workspace.yamlName} == 'true' }}`, - ' needs:', - ' - changes', - ].join('\n') + '\n'); - const deps = workspace.deepDependencies; - for (let dep of deps.sort()) { - if (dep == workspace.name) continue; - workflowStream.write(` - ${workspaces[dep].yamlName}\n`); - } - } -} - -async function generateWorkspaceWorkflow(workspace: Workspace): Promise { - const workflowMeta = getWorkspaceWorkflowMeta(workspace); - const workflowPath = path.join('.github', 'workflows', workflowMeta.filename); - if (fs.existsSync(workflowPath)) { - console.log(`Not overwriting existing workflow: ${workflowMeta.filename}`); - return; - } - let workflowFileHandle: fs.promises.FileHandle | undefined; - try { - workflowFileHandle = await fs.promises.open(workflowPath, 'w'); - const workspaceWorkflow = workspaceTemplate - .replace(/WS_NAME/g, workspace.name) - .replace(/WS_LOCATION/g, workspace.location); - workflowFileHandle.write(Buffer.from(workspaceWorkflow, 'utf8')); - } finally { - workflowFileHandle?.close(); - } -} - -const main = async () => { - console.log('Querying workspaces...'); - const packages = JSON.parse((await execAsync('npm query .workspace')).stdout) as Array; - console.log('Calculating dependencies...'); - const workspaces = calculateDependencies(packages, [DependencyType.Dependencies]); - console.log('Sorting modules in dependency order...'); - const sortedWorkspaces = sortWorkspaces(workspaces); - console.log('Generating main workflow...'); - fs.mkdirSync(path.join('.github', 'workflows'), {recursive: true}); - await generateWorkflow(sortedWorkspaces, workspaces); - console.log('Generating stub workflows for workspaces...'); - for (let workspace of sortedWorkspaces) { - await generateWorkspaceWorkflow(workspace); - } -}; - -main().then( - () => { - console.log('Done.'); - process.exitCode = 0; - }, - e => { - console.error(e); - process.exitCode = 1; - } -); diff --git a/scripts/update-gha-workflows.ts b/scripts/update-gha-workflows.ts new file mode 100644 index 00000000000..ded1bbc0858 --- /dev/null +++ b/scripts/update-gha-workflows.ts @@ -0,0 +1,271 @@ +// update-gha-workflows.ts — Regenerate CI files that reference individual workspaces. +// +// Fully regenerated: +// .github/path-filters.yml — per-workspace path filters (global paths + transitive deps) +// +// Incrementally updated (new entries added, existing ones preserved): +// .github/workflows/publish.yml — per-workspace npm publish steps +// +// Usage: +// npx ts-node scripts/update-gha-workflows.ts +// npm run refresh-gh-workflow + +import {exec} from 'child_process'; +import fs from 'fs'; +import path from 'path'; +import {promisify} from 'util'; + +const execAsync = promisify(exec); + +// ─── Types ─────────────────────────────────────────────────────────────────── + +type PackageJson = { + name: string; + location: string; + dependencies?: Record; + devDependencies?: Record; + peerDependencies?: Record; +}; + +type Workspace = { + /** Full npm name, e.g. "@scratch/scratch-vm" */ + name: string; + /** Bare name, e.g. "scratch-vm" */ + bareName: string; + /** Relative dir, e.g. "packages/scratch-vm" */ + location: string; + /** Direct workspace dependencies (production only) */ + directDeps: string[]; + /** Transitive closure of workspace dependencies (production only), including self */ + deepDeps: string[]; +}; + +// ─── Dependency resolution ─────────────────────────────────────────────────── + +function resolveWorkspaces(packages: PackageJson[]): Workspace[] { + const nameSet = new Set(packages.map(p => p.name)); + + // Build workspace records with direct deps + const workspaceMap = new Map(); + for (const pkg of packages) { + const bareName = path.basename(pkg.location); + const directDeps = Object.keys(pkg.dependencies ?? {}).filter(d => nameSet.has(d)); + workspaceMap.set(pkg.name, { + name: pkg.name, + bareName, + location: pkg.location, + directDeps, + deepDeps: [], + }); + } + + // Compute transitive closure (production deps only) + function collectDeep(name: string, visited: Set): void { + if (visited.has(name)) return; + visited.add(name); + const ws = workspaceMap.get(name); + if (!ws) return; + for (const dep of ws.directDeps) { + collectDeep(dep, visited); + } + } + + for (const ws of workspaceMap.values()) { + const visited = new Set(); + collectDeep(ws.name, visited); + ws.deepDeps = [...visited]; + } + + // Preserve the order from the root package.json workspaces array + return [...workspaceMap.values()]; +} + +// ─── path-filters.yml generation ───────────────────────────────────────────── + +const PATH_FILTERS_HEADER = `\ +# This file is generated by scripts/update-gha-workflows.ts +# +# To regenerate: npm run refresh-gh-workflow +`; + +const GLOBAL_PATHS = [ + '.github/path-filters.yml', + '.github/workflows/ci.yml', + '.nvmrc', + 'package.json', + 'package-lock.json', + 'scripts/**', +]; + +function generatePathFilters(workspaces: Workspace[], workspaceMap: Map): string { + const lines: string[] = []; + + // Header comment + lines.push(PATH_FILTERS_HEADER); + + // global anchor + lines.push('# The `&global` anchor defines a set of common paths to include by reference in the other filters.'); + lines.push('global: &global'); + for (const p of GLOBAL_PATHS) { + lines.push(` - "${p}"`); + } + lines.push(''); + + // any-workspace + lines.push('any-workspace:'); + lines.push(' - *global'); + lines.push(' - "packages/**"'); + lines.push(''); + + // Per-workspace entries, each including self + all transitive production deps + for (const ws of workspaces) { + lines.push(`${ws.bareName}:`); + lines.push(' - *global'); + + // Collect all package directories (self + transitive deps), sorted for stability + const depLocations = ws.deepDeps + .map(depName => workspaceMap.get(depName)!) + .map(dep => dep.location) + .sort(); + + for (const loc of depLocations) { + lines.push(` - "${loc}/**"`); + } + } + + lines.push(''); + return lines.join('\n'); +} + +// ─── publish.yml update ────────────────────────────────────────────────────── + +const PUBLISH_STEP_MARKER = ' - name: Push to Develop'; + +/** + * Scans publish.yml for existing "Publish " steps and adds a default + * publish step for any workspace that doesn't already have one. + * + * Packages that should NOT be auto-published (virtual packages, meta-packages, + * packages with special publish logic already present) are skipped if they + * already have a step — they are never removed. + */ +// TODO: Add exclude list for packages that we don't want to auto-publish, +// as they are introduced to the monorepo. +function updatePublishWorkflow(content: string, workspaces: Workspace[]): string { + // Find all existing "- name: Publish " step names + const existingPublishSteps = new Set(); + const publishStepRegex = /- name: Publish (.+)$/gm; + let match; + while ((match = publishStepRegex.exec(content)) !== null) { + existingPublishSteps.add(match[1].trim()); + } + + // Determine which workspaces need a new publish step + const missing: Workspace[] = []; + for (const ws of workspaces) { + // Check if there's already a publish step matching this workspace's bare name + if (existingPublishSteps.has(ws.bareName)) continue; + // Also check for any step that mentions the full @scratch/ name in its title + if (existingPublishSteps.has(ws.name)) continue; + missing.push(ws); + } + + if (missing.length === 0) { + return content; + } + + // Build the new publish steps + const newSteps = missing.map(ws => { + return [ + ` - name: Publish ${ws.bareName}`, + ` run: npm publish --access=public --tag="$NPM_TAG" --ignore-scripts --workspace=${ws.name}`, + ].join('\n'); + }).join('\n\n'); + + // Insert before "Push to Develop" + const insertIndex = content.indexOf(PUBLISH_STEP_MARKER); + if (insertIndex === -1) { + console.warn('Warning: Could not find "Push to Develop" step in publish.yml.'); + console.warn('New publish steps were NOT added. Please add them manually:'); + for (const ws of missing) { + console.warn(` - Publish ${ws.bareName}`); + } + return content; + } + + return content.slice(0, insertIndex) + newSteps + '\n\n' + content.slice(insertIndex); +} + +// ─── Main ──────────────────────────────────────────────────────────────────── + +async function main(): Promise { + const repoRoot = path.resolve(__dirname, '..'); + const ghDir = path.join(repoRoot, '.github'); + const workflowsDir = path.join(ghDir, 'workflows'); + + // 1. Query workspaces + console.log('Querying workspaces...'); + const {stdout} = await execAsync('npm query .workspace'); + const packages = JSON.parse(stdout) as PackageJson[]; + console.log(` Found ${packages.length} workspaces`); + + // 2. Resolve dependencies + console.log('Resolving transitive dependencies...'); + const resolvedWorkspaces = resolveWorkspaces(packages); + const workspaceMap = new Map(resolvedWorkspaces.map(ws => [ws.name, ws])); + + // Sort by the declared order in the root package.json workspaces array + // (npm query .workspace returns packages alphabetically, not in declaration order) + const rootPackageJson = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8')); + const declaredOrder: string[] = (rootPackageJson.workspaces as string[]).map( + (ws: string) => path.basename(ws), + ); + const workspaces = resolvedWorkspaces.sort( + (a, b) => declaredOrder.indexOf(a.bareName) - declaredOrder.indexOf(b.bareName), + ); + + for (const ws of workspaces) { + const depNames = ws.deepDeps.filter(d => d !== ws.name).map(d => workspaceMap.get(d)!.bareName); + if (depNames.length > 0) { + console.log(` ${ws.bareName} → ${depNames.join(', ')}`); + } else { + console.log(` ${ws.bareName} (no workspace deps)`); + } + } + + // 3. Regenerate path-filters.yml + console.log('Generating .github/path-filters.yml...'); + const pathFiltersPath = path.join(ghDir, 'path-filters.yml'); + const pathFiltersContent = generatePathFilters(workspaces, workspaceMap); + fs.mkdirSync(ghDir, {recursive: true}); + fs.writeFileSync(pathFiltersPath, pathFiltersContent, 'utf8'); + console.log(` Wrote ${pathFiltersPath}`); + + // 4. Update publish.yml + const publishPath = path.join(workflowsDir, 'publish.yml'); + if (fs.existsSync(publishPath)) { + console.log('Updating .github/workflows/publish.yml...'); + const originalPublish = fs.readFileSync(publishPath, 'utf8'); + const updatedPublish = updatePublishWorkflow(originalPublish, workspaces); + if (updatedPublish !== originalPublish) { + fs.writeFileSync(publishPath, updatedPublish, 'utf8'); + console.log(' Added new publish steps.'); + } else { + console.log(' No changes needed.'); + } + } else { + console.warn('Warning: .github/workflows/publish.yml not found. Skipping publish step generation.'); + } + + console.log('Done.'); +} + +main().then( + () => { + process.exitCode = 0; + }, + e => { + console.error(e); + process.exitCode = 1; + }, +); diff --git a/scripts/workspace-template.yml b/scripts/workspace-template.yml deleted file mode 100644 index 22e91187cbe..00000000000 --- a/scripts/workspace-template.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: "Workspace: WS_NAME" - -on: - workflow_call: - workflow_dispatch: - -concurrency: - group: 'WS_NAME @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' - -permissions: - contents: write # publish a GitHub release - pages: write # deploy to GitHub Pages - issues: write # comment on released issues - pull-requests: write # comment on released pull requests - -jobs: - ci-cd: - runs-on: ubuntu-latest - defaults: - run: - working-directory: "WS_LOCATION" - environment: >- - ${{ - ( - ( - (github.ref == 'refs/heads/main') || - (github.ref == 'refs/heads/master') - ) && 'production' - ) || - ( - ( - (github.ref == 'refs/heads/beta') || - (github.ref == 'refs/heads/develop') || - startsWith(github.ref, 'refs/heads/hotfix/') || - startsWith(github.ref, 'refs/heads/release/') - ) && 'staging' - ) || - '' - }} - env: - SCRATCH_SHOULD_DEPLOY: ${{ vars.SCRATCH_ENV != '' }} - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - cache: 'npm' - node-version-file: '.nvmrc' - - name: Debug info - run: | - cat <' }} - Node version: $(node --version) - NPM version: $(npm --version) - GitHub ref: ${{ github.ref }} - GitHub head ref: ${{ github.head_ref }} - Working directory: $(pwd) - EOF - - uses: ./.github/actions/install-dependencies - - name: Build - run: npm run build - - name: Test - run: npm run test - - name: Release - if: ${{ env.SCRATCH_SHOULD_DEPLOY == 'true' }} - env: - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: npx --no -- semantic-release