fast test #401
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Compile and release | |
| on: push | |
| jobs: | |
| prebuild: | |
| runs-on: rehosting-arc | |
| outputs: | |
| targets: ${{ steps.find_targets.outputs.targets }} | |
| versions: ${{ steps.find_targets.outputs.versions }} | |
| sources_dir: ${{ steps.setup_sources.outputs.sources_dir }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.ref }} | |
| - name: Log git revisions of all linux projects | |
| run: | | |
| echo "Main repo revision:" && git rev-parse HEAD | |
| echo | |
| echo "Submodule revisions:" && git submodule status | |
| echo | |
| echo "Full submodule SHAs:" && git submodule foreach 'echo $name: $(git rev-parse HEAD)' | |
| - name: Ensure local bare clone of base Linux repo | |
| run: | | |
| set -eux | |
| BASE_REPO_DIR="/home/runner/_shared/linux" | |
| BASE_REPO_URL="https://github.com/rehosting/linux" | |
| # Clone bare base repo if missing | |
| if [ ! -d "$BASE_REPO_DIR" ]; then | |
| echo "Cloning bare base repo to $BASE_REPO_DIR" | |
| git clone --bare "$BASE_REPO_URL" "$BASE_REPO_DIR" | |
| cd $BASE_REPO_DIR && git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" | |
| fi | |
| # Always fetch latest from upstream | |
| cd "$BASE_REPO_DIR" && git fetch origin --prune --tags --force | |
| - name: Ensure linux cache exists | |
| run: | | |
| BASE_CACHE_DIR="/home/runner/_shared/linux_builder/cache" | |
| if [ ! -d "$BASE_CACHE_DIR" ]; then | |
| mkdir -p "$BASE_CACHE_DIR" | |
| fi | |
| - name: Install rsync | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y rsync | |
| - name: Setup shared Linux kernel sources | |
| id: setup_sources | |
| run: | | |
| set -eux | |
| SOURCES_DIR="/home/runner/_shared/linux_sources/" | |
| echo "Using stable source directory: $SOURCES_DIR" | |
| echo "sources_dir=$SOURCES_DIR" >> $GITHUB_OUTPUT | |
| # Ensure the stable directory exists and copy the entire repo into it. | |
| # The --delete flag keeps the destination in sync with the source. | |
| mkdir -p "$SOURCES_DIR" | |
| rsync -a --delete . "$SOURCES_DIR/" | |
| # Change into the stable directory to perform all subsequent git operations | |
| cd "$SOURCES_DIR" | |
| BASE_REPO_DIR="/home/runner/_shared/linux" | |
| sed -i "s|url = https://github.com/rehosting/linux.git|url = file://$BASE_REPO_DIR|g" .gitmodules | |
| # Sync and update submodules from within the stable repository | |
| git submodule sync | |
| GIT_ALLOW_PROTOCOL=file:https git submodule update --init --depth 1 --jobs 2 | |
| # Use rsync to move the linux directory into the stable location | |
| # This is more robust than mv and helps preserve attributes. | |
| rsync -a --delete linux/ "$SOURCES_DIR/linux/" | |
| - name: Find valid targets and versions sets | |
| id: find_targets | |
| run: | | |
| TARGETS_SET=() | |
| VERSIONS_SET=() | |
| for version_dir in configs/*/; do | |
| version=$(basename "$version_dir") | |
| VERSIONS_SET+=("$version") | |
| for config_file in "$version_dir"*; do | |
| if [[ -f "$config_file" && ! "$config_file" =~ \.inc$ && ! "$config_file" =~ \.unused$ ]]; then | |
| target=$(basename "$config_file") | |
| TARGETS_SET+=("$target") | |
| fi | |
| done | |
| done | |
| UNIQUE_TARGETS=$(printf "%s\n" "${TARGETS_SET[@]}" | sort -u | awk '{printf "\"%s\",",$0}' | sed 's/,$//') | |
| UNIQUE_VERSIONS=$(printf "%s\n" "${VERSIONS_SET[@]}" | sort -u | awk '{printf "\"%s\",",$0}' | sed 's/,$//') | |
| TARGETS_OUTPUT="[${UNIQUE_TARGETS}]" | |
| VERSIONS_OUTPUT="[${UNIQUE_VERSIONS}]" | |
| echo "targets=$TARGETS_OUTPUT" >> $GITHUB_OUTPUT | |
| echo "versions=$VERSIONS_OUTPUT" >> $GITHUB_OUTPUT | |
| echo "Found valid targets: $TARGETS_OUTPUT" | |
| echo "Found valid versions: $VERSIONS_OUTPUT" | |
| build: | |
| needs: prebuild | |
| runs-on: rehosting-arc | |
| if: github.event.pull_request.draft == false | |
| strategy: | |
| matrix: | |
| target_version: ${{ fromJSON(needs.prebuild.outputs.targets) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.ref }} | |
| - name: Extract target and version | |
| id: extract | |
| run: | | |
| TARGET="${{ matrix.target_version }}" | |
| echo "target=$TARGET" >> $GITHUB_OUTPUT | |
| echo "Building target: $TARGET" | |
| - name: Trust Harbor's self-signed certificate | |
| run: | | |
| echo "Fetching certificate from ${{ secrets.REHOSTING_ARC_REGISTRY }}" | |
| openssl s_client -showcerts -connect ${{ secrets.REHOSTING_ARC_REGISTRY }}:443 < /dev/null 2>/dev/null | openssl x509 -outform PEM | sudo tee /usr/local/share/ca-certificates/harbor.crt > /dev/null | |
| sudo update-ca-certificates | |
| - name: Log in to Rehosting Arc Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{secrets.REHOSTING_ARC_REGISTRY}} | |
| username: ${{ secrets.REHOSTING_ARC_REGISTRY_USER }} | |
| password: ${{ secrets.REHOSTING_ARC_REGISTRY_PASSWORD }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| driver-opts: | | |
| image=moby/buildkit:master | |
| network=host | |
| buildkitd-config-inline: | | |
| [registry."${{ secrets.REHOSTING_ARC_REGISTRY }}"] | |
| insecure = true | |
| http = true | |
| - name: Build kernel_builder docker image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| push: false | |
| tags: | | |
| rehosting/linux_builder:latest | |
| build-args: | | |
| REGISTRY=${{ secrets.REHOSTING_ARC_REGISTRY }}/proxy | |
| TARGET=${{ matrix.target_version }} | |
| cache-from: | | |
| type=registry,ref=${{secrets.REHOSTING_ARC_REGISTRY}}/rehosting/linux_builder:${{ matrix.target_version }}_cache,mode=max | |
| cache-to: | | |
| type=registry,ref=${{secrets.REHOSTING_ARC_REGISTRY}}/rehosting/linux_builder:${{ matrix.target_version }}_cache,mode=max | |
| outputs: type=docker | |
| - name: Build Kernel for ${{ matrix.target_version }} | |
| run: | | |
| set -eux | |
| TARGET="${{ matrix.target_version }}" | |
| VERSIONS_JSON='${{ needs.prebuild.outputs.versions }}' | |
| # BASE_CACHE_DIR="/home/runner/_shared/linux_builder/cache" | |
| # Use the output from the prebuild job | |
| SOURCES_DIR="${{ needs.prebuild.outputs.sources_dir }}/linux" | |
| if [ -z "$VERSIONS_JSON" ] || [ "$VERSIONS_JSON" = "[]" ]; then | |
| VERSIONS="" | |
| else | |
| VERSIONS=$(echo "$VERSIONS_JSON" | jq -r '.[]' | xargs) | |
| fi | |
| # Use a run-specific output directory to avoid clashes | |
| BUILD_OUTPUT="/home/runner/_shared/runs/$GITHUB_RUN_ID/build-output" | |
| mkdir -p $BUILD_OUTPUT | |
| ls -l "$BUILD_OUTPUT" | |
| mount | |
| :> $BUILD_OUTPUT/kernels-latest-${TARGET}.tar.gz | |
| :> $BUILD_OUTPUT/kernel-devel-all-${TARGET}.tar.gz | |
| aggregate: | |
| if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch' | |
| needs: build | |
| runs-on: rehosting-arc | |
| env: | |
| MATRIX_VERSIONS: ${{ toJSON(needs.build.strategy.matrix.version) }} | |
| permissions: | |
| actions: write | |
| contents: write | |
| steps: | |
| - name: Trust Harbor's self-signed certificate | |
| run: | | |
| echo "Fetching certificate from ${{ secrets.REHOSTING_ARC_REGISTRY }}" | |
| openssl s_client -showcerts -connect ${{ secrets.REHOSTING_ARC_REGISTRY }}:443 < /dev/null 2>/dev/null | openssl x509 -outform PEM | sudo tee /usr/local/share/ca-certificates/harbor.crt > /dev/null | |
| sudo update-ca-certificates | |
| - name: Log in to Rehosting Arc Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{secrets.REHOSTING_ARC_REGISTRY}} | |
| username: ${{ secrets.REHOSTING_ARC_REGISTRY_USER }} | |
| password: ${{ secrets.REHOSTING_ARC_REGISTRY_PASSWORD }} | |
| - name: Combine all kernels into a single archive | |
| run: | | |
| set -eux | |
| RUNS_PARENT="/home/runner/_shared/runs" | |
| RUNS_DIR="$RUNS_PARENT/$GITHUB_RUN_ID" | |
| BUILD_OUTPUT="$RUNS_DIR/build-output" | |
| echo "[DEBUG] Listing available per-target kernel archives:" | |
| find "$BUILD_OUTPUT" -maxdepth 1 -name "kernels-latest-*.tar.gz" -print || true | |
| rm -rf combined-kernels && mkdir combined-kernels | |
| for archive in "$BUILD_OUTPUT"/kernels-latest-*.tar.gz; do | |
| [ -e "$archive" ] || continue | |
| echo "[DEBUG] Extracting $archive into combined-kernels" | |
| tar -xzf "$archive" -C combined-kernels | |
| done | |
| echo "[DEBUG] Contents of combined-kernels after extraction:" | |
| find combined-kernels || true | |
| # Merge osi.config for every detected version directory | |
| if [ -d combined-kernels/kernels ]; then | |
| for vdir in combined-kernels/kernels/*; do | |
| [ -d "$vdir" ] || continue | |
| version=$(basename "$vdir") | |
| echo "[DEBUG] Merging osi.config for version $version" | |
| { | |
| for archive in "$BUILD_OUTPUT"/kernels-latest-*.tar.gz; do | |
| [ -e "$archive" ] || continue | |
| tar -O -xf "$archive" "kernels/$version/osi.config" 2>/dev/null || true | |
| done | |
| } > "combined-kernels/kernels/$version/osi.config" | |
| done | |
| fi | |
| tar -czvf kernels-latest.tar.gz -C combined-kernels . | |
| - name: Aggregate all kernel-devel artifacts | |
| run: | | |
| set -eux | |
| RUNS_PARENT="/home/runner/_shared/runs" | |
| RUNS_DIR="$RUNS_PARENT/$GITHUB_RUN_ID" | |
| BUILD_OUTPUT="$RUNS_DIR/build-output" | |
| mkdir -p kernel-devel-all | |
| for archive in "$BUILD_OUTPUT"/kernel-devel-all-*.tar.gz; do | |
| [ -e "$archive" ] || continue | |
| echo "[DEBUG] Extracting $archive into kernel-devel-all/" | |
| tar -xzf "$archive" -C kernel-devel-all | |
| done | |
| tar -czvf kernel-devel-all.tar.gz -C kernel-devel-all . | |
| - name: Create and publish release | |
| uses: softprops/action-gh-release@v1 | |
| with: | |
| files: | | |
| kernels-latest.tar.gz | |
| kernel-devel-all.tar.gz | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| tag_name: ${{ github.ref_name }} | |
| - name: Cleanup per-run kernel clones | |
| if: always() | |
| run: | | |
| RUNS_PARENT="/home/runner/_shared/runs" | |
| RUNS_DIR="$RUNS_PARENT/$GITHUB_RUN_ID" | |
| echo "Cleaning up kernel clones in $RUNS_DIR" | |
| rm -rf "$RUNS_DIR" |