diff --git a/.github/actions/aws_s3_helper/action.yml b/.github/actions/aws_s3_helper/action.yml deleted file mode 100644 index 8e430072..00000000 --- a/.github/actions/aws_s3_helper/action.yml +++ /dev/null @@ -1,199 +0,0 @@ -name: AWS S3 Helper -description: Upload and download files from AWS S3 - -inputs: - s3_bucket: - description: S3 Bucket Name - required: true - local_file: - description: Local file paths. For 'multi-upload', this should be a path to a file containing a list of local file paths (one per line). For 'single-upload', it's a single local file path. - required: false - default: ../artifacts/file_list.txt - download_file: - description: S3 key of the file to download (e.g., path/to/your/file.txt). Used only in 'download' mode. - required: false - default: '' - download_location: - description: Local directory or file path where the downloaded file should be placed. Used only in 'download' mode. - required: false - default: . - artifacts: - description: > - Optional S3 key or prefix for upload/download. - - If ends with '/', it is treated as a prefix and the local filename is appended. - - If not ending with '/', it is treated as a full S3 key. - - If empty, a default location is derived. - required: false - default: '' - mode: - description: Mode of operation (single-upload, multi-upload, download) - required: true - default: single-upload - deviceTree: - description: Target device Tree name or identifier, used to organize uploads in S3. - type: string - required: true - -outputs: - presigned_url: - description: Pre-signed URL for the single uploaded file (only available in 'single-upload' mode). - value: ${{ steps.sync-data.outputs.presigned_url }} - -runs: - using: "composite" - steps: - - name: Sync Data - id: sync-data - shell: bash - env: - UPLOAD_LOCATION: ${{ github.repository_owner }}/${{ github.event.repository.name }}/${{ github.workflow }}/${{ github.head_ref != '' && github.head_ref || github.run_id }}/${{ inputs.deviceTree }}/ - DEFAULT_UPLOAD_LOCATION: ${{ github.repository_owner }}/${{ github.event.repository.name }}/ - run: | - set -euo pipefail - echo "::group::$(printf '__________ %-100s' 'Process' | tr ' ' _)" - - S3_BUCKET="${{ inputs.s3_bucket }}" - MODE="${{ inputs.mode }}" - LOCAL_FILE="${{ inputs.local_file }}" - ARTIFACTS="${{ inputs.artifacts }}" - DOWNLOAD_FILE="${{ inputs.download_file }}" - DOWNLOAD_LOCATION="${{ inputs.download_location }}" - WORKSPACE="${{ github.workspace }}" - DEVICE="${{ inputs.deviceTree }}" - UPLOAD_LOCATION="${{ env.UPLOAD_LOCATION }}" - DEFAULT_UPLOAD_LOCATION="${{ env.DEFAULT_UPLOAD_LOCATION }}" - - # Helpers - _choose_upload_key() { - # $1: local path of file to upload - local local_path="$1" - local base; base="$(basename "$local_path")" - - if [ -n "$ARTIFACTS" ]; then - # user provided a path; decide whether it's a prefix or a full key - if [[ "$ARTIFACTS" == */ ]]; then - # prefix: append filename - echo "${ARTIFACTS}${base}" - else - # full key - echo "${ARTIFACTS}" - fi - else - # default to computed upload location + filename - echo "${UPLOAD_LOCATION}${base}" - fi - } - - case "${MODE}" in - multi-upload) - echo "Uploading files to S3 bucket (multi-upload)..." - if [ ! -f "$LOCAL_FILE" ]; then - echo "ERROR: File list not found: $LOCAL_FILE" >&2 - exit 1 - fi - - mkdir -p "${WORKSPACE}/${DEVICE}" - MANIFEST="${WORKSPACE}/${DEVICE}/presigned_urls_${DEVICE}.json" - echo "{" > "$MANIFEST" - first_line=true - - while IFS= read -r file || [ -n "$file" ]; do - # Skip blank and comment lines - [[ -z "${file// }" || "$file" =~ ^[[:space:]]*# ]] && continue - - if [ -f "$file" ]; then - key="$(_choose_upload_key "$file")" - s3_uri="s3://${S3_BUCKET}/${key}" - - echo "Uploading $file to $s3_uri ..." - aws s3 cp "$file" "$s3_uri" --sse AES256 - echo "Uploaded $file to $s3_uri" - - echo "Creating Pre-signed URL for $s3_uri ..." - presigned_url=$(aws s3 presign "$s3_uri" --expires-in 259200) - - if [ "$first_line" = true ]; then - first_line=false - else - echo "," >> "$MANIFEST" - fi - - printf ' "%s": "%s"' "$file" "$presigned_url" >> "$MANIFEST" - echo "" - else - echo "Warning: $file does not exist or is not a regular file." >&2 - fi - done < "$LOCAL_FILE" - - - echo "}" >> "$MANIFEST" - ;; - - single-upload) - echo "Uploading single file to S3 bucket..." - if [ ! -f "$LOCAL_FILE" ]; then - echo "ERROR: Local file not found: $LOCAL_FILE" >&2 - exit 1 - fi - - key="$(_choose_upload_key "$LOCAL_FILE")" - s3_uri="s3://${S3_BUCKET}/${key}" - - echo "Uploading $LOCAL_FILE to $s3_uri ..." - aws s3 cp "$LOCAL_FILE" "$s3_uri" --sse AES256 - echo "Uploaded $LOCAL_FILE to $s3_uri" - - echo "Creating Pre-signed URL for $s3_uri ..." - presigned_url=$(aws s3 presign "$s3_uri" --expires-in 259200) - echo "presigned_url=${presigned_url}" >> "$GITHUB_OUTPUT" - ;; - - download) - - echo "Downloading file from S3 bucket..." - # Priority: - # - If ARTIFACTS is provided and not ending with '/', treat it as key - # - Else if ARTIFACTS ends with '/', error (need a concrete key) - # - Else use DOWNLOAD_FILE (must be set) - if [ -n "$ARTIFACTS" ]; then - if [[ "$ARTIFACTS" == */ ]]; then - echo "ERROR: 'artifacts' ends with '/'. Please provide a full S3 key (not a prefix) for download mode, or use 'download_file'." >&2 - exit 1 - fi - key="$ARTIFACTS" - else - if [ -z "$DOWNLOAD_FILE" ]; then - echo "ERROR: 'download_file' must be provided when 'artifacts' is empty in download mode." >&2 - exit 1 - fi - key="$DOWNLOAD_FILE" - fi - s3_uri="s3://${S3_BUCKET}/${key}" - - # Ensure download location exists if it is a directory - if [ -d "$DOWNLOAD_LOCATION" ]; then - echo "Downloading $s3_uri -> ${DOWNLOAD_LOCATION}/" - aws s3 cp "$s3_uri" "${DOWNLOAD_LOCATION}/" - else - mkdir -p "$(dirname "$DOWNLOAD_LOCATION")" - echo "Downloading $s3_uri -> ${DOWNLOAD_LOCATION}" - aws s3 cp "$s3_uri" "${DOWNLOAD_LOCATION}" - fi - - echo "Downloaded ${s3_uri} to ${DOWNLOAD_LOCATION}" - ;; - *) - echo "Invalid mode. Use 'upload' or 'download'." - exit 1 - ;; - esac - - echo "::endgroup::" - - - name: Upload artifacts - if: ${{ inputs.mode == 'multi-upload' }} - uses: actions/upload-artifact@v4 - with: - name: presigned_urls_${{ inputs.deviceTree }}.json - path: ${{ github.workspace }}/${{ inputs.deviceTree }}/presigned_urls_${{ inputs.deviceTree }}.json - retention-days: 3 diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml deleted file mode 100644 index f64628c9..00000000 --- a/.github/actions/build/action.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Build workspace -description: | - Builds and packages the FastRPC workspace. - This composite action compiles the FastRPC source code and kernel modules - using a specified Docker image, verifies the compilation artifacts, and - then packages the resulting binaries, libraries, and kernel modules - into a gzipped CPIO ramdisk. It also includes steps to copy - firmware files and test binaries into the ramdisk. - Finally, it unpacks the generated ramdisk for inspection and verification. - -inputs: - docker_image: - description: Docker image - required: true - default: fastrpc-image:latest - workspace_path: - description: Workspace path - required: true - deviceTree: - description: Target device Tree name or identifier, used to organize uploads in S3. - type: string - required: true - -runs: - using: "composite" - steps: - - name: Compile fastrpc code using Docker Image - shell: bash - run: | - cd ${{ inputs.workspace_path }} - echo "::group::$(printf '__________ %-100s' 'Compile fastrpc' | tr ' ' _)" - docker run -i --rm \ - --user $(id -u):$(id -g) \ - --workdir="$PWD" \ - -v "$(dirname $PWD)":"$(dirname $PWD)" \ - ${{ inputs.docker_image }} bash -c " - ./gitcompile --host=aarch64-linux-gnu - make install DESTDIR=${{ inputs.workspace_path }}/artifacts/ramdisk_fastrpc - " - echo "::endgroup::" - - - name: Download Kernel artifact - uses: qualcomm/fastrpc/.github/actions/aws_s3_helper@development - with: - s3_bucket: qli-prd-fastrpc-gh-artifacts - artifacts: kernel-artifact/kobj.tar.gz - deviceTree: false - download_location: ${{ inputs.workspace_path }} - mode: download - - - name: Package Files into ramdisk - shell: bash - run: | - echo "Extracting kobj.tar.gz" - ls -la ${{ inputs.workspace_path }}/ - mkdir -p extracted_kobj - tar -xzf "${{ inputs.workspace_path }}/kobj.tar.gz" -C extracted_kobj - echo "Contents of extracted_kobj:" - find extracted_kobj -type d - # Move extracted folder to kobj - rm -rf kobj - mv extracted_kobj/kobj kobj - ls -la kobj - - echo "Package FastRPC firmware files" - cp -r ${{ inputs.workspace_path }}/firmware_dir/* ${{ inputs.workspace_path }}/artifacts/ramdisk_fastrpc/ - - echo "Package DLKM into ramdisk" - cd ${{ inputs.workspace_path }}/kobj/tar-install - cp -r lib/* ${{ inputs.workspace_path }}/artifacts/ramdisk_fastrpc/usr/lib/ - ls -la ${{ inputs.workspace_path }}/artifacts/ramdisk_fastrpc/usr/lib/ - - cd ${{ inputs.workspace_path }}/artifacts/ - cd ramdisk_fastrpc - find . | cpio -o -H newc > ../ramdisk_fastrpc.cpio - cd .. - gzip -9 ramdisk_fastrpc.cpio - mv ramdisk_fastrpc.cpio.gz ramdisk_fastrpc.gz - - - name: Unpack and inspect ramdisk - shell: bash - run: | - echo "Unpack and mount ramdisk" - cd ${{ inputs.workspace_path }}/artifacts - mkdir -p ramdisk_test - gunzip -c ramdisk_fastrpc.gz > ramdisk_test/ramdisk - - echo "Unpacking ramdisk" - cd ramdisk_test - # Use safer syntax to avoid broken pipe - cpio -idmv < ramdisk || echo "cpio unpack failed" - - # Inspect contents - ls -ltr usr/lib | grep rpc || true - ls -ltr usr/bin | grep rpc || true diff --git a/.github/actions/build_docker_image/action.yml b/.github/actions/build_docker_image/action.yml deleted file mode 100644 index 7a1787e2..00000000 --- a/.github/actions/build_docker_image/action.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: Build fastrpc docker image -description: | - Build fastrpc docker image - This GitHub Action builds the fastrpc Docker image using a multi-step process: - Checks out the fastrpc-image repository to obtain the Dockerfile. - Sets up Docker Buildx for advanced build capabilities. - Captures the current user's UID, GID, and username for use as Docker build arguments. - Builds the Docker image with caching enabled, tagging it as 'fastrpc-image:latest'. - Outputs a success message and the image name after the build. - Re-checks out the main repository for any subsequent workflow steps. - -inputs: - image: - description: The docker image to Build - required: true - default: fastrpc-image:latest - -runs: - using: "composite" - steps: - # Checkout fastrpc-image repository to get the Dockerfile - - name: Checkout fastrpc-image repo for Dockerfile - uses: actions/checkout@v4 - with: - fetch-depth: 0 - repository: qualcomm/fastrpc-image - ref: main - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Pre-Build Docker Image - id: pre-build - shell: bash - run: | - uid=$(id -u) - gid=$(id -g) - user=$(whoami) - echo "uid=$uid" >> "$GITHUB_OUTPUT" - echo "gid=$gid" >> "$GITHUB_OUTPUT" - echo "user=$user" >> "$GITHUB_OUTPUT" - echo "Running as user: $user (UID: $uid, GID: $gid)" - - - name: Build fastrpc Docker Image - id: build - uses: docker/build-push-action@v6 - with: - context: . - push: false - load: true - tags: fastrpc-image:latest - build-args: | - "USER=${{ steps.pre-build.outputs.user }}" - "UID=${{ steps.pre-build.outputs.uid }}" - "GID=${{ steps.pre-build.outputs.gid }}" - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Post-Build Docker Image - id: post-build - shell: bash - run: | - echo "Docker image fastrpc-image:latest built successfully." - echo "image_name=fastrpc-image:latest" >> $GITHUB_OUTPUT - - # Checkout the main repo again for later steps - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.ref }} \ No newline at end of file diff --git a/.github/actions/lava_job_render/action.yml b/.github/actions/lava_job_render/action.yml deleted file mode 100644 index 8b18d2eb..00000000 --- a/.github/actions/lava_job_render/action.yml +++ /dev/null @@ -1,184 +0,0 @@ -name: Test Action -description: | - This GitHub Action composite action prepares data for LAVA job definitions. - It reads presigned URLs from a JSON file, creates a metadata.json file, - uploads it to S3, and then populates a cloudData.json file with various - artifact URLs (image, vmlinux, firmware, modules, dtb, metadata, ramdisk). - Finally, it generates a LAVA job definition using these populated data files. - -inputs: - docker_image: - description: Docker image - required: true - default: fastrpc-image:latest - -runs: - using: "composite" - steps: - - name: Process presigned_urls.json - id: process_urls - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - const p = require('path'); - // Helper function to find URL by filename - function findUrlByFilename(filename) { - for (const [path, url] of Object.entries(data)) { - if (path.endsWith(filename)) { - return url; - } - } - return null; - } - const filePath = p.join( - process.env.GITHUB_WORKSPACE, - `presigned_urls_${process.env.DEVICE_TREE}.json` - ); - - if (fs.existsSync(filePath)) { - console.log("File exists"); - } else { - console.log("File does not exist"); - core.setFailed(`File not found: ${filePath}`); - } - // Read the JSON file - const data = JSON.parse(fs.readFileSync(filePath, 'utf-8')); - // Extract URLs into variables - const modulesTarUrl = findUrlByFilename('modules.tar.xz'); - const imageUrl = findUrlByFilename('Image'); - const vmlinuxUrl = findUrlByFilename('vmlinux'); - const firmwareUrl = findUrlByFilename('ramdisk_fastrpc.gz'); - const dtbFilename = `${process.env.DEVICE_TREE}.dtb`; - const dtbUrl = findUrlByFilename(dtbFilename); - // Set outputs - core.setOutput('modules_url', modulesTarUrl); - core.setOutput('image_url', imageUrl); - core.setOutput('vmlinux_url', vmlinuxUrl); - core.setOutput('firmware_url', firmwareUrl); - core.setOutput('dtb_url', dtbUrl); - console.log(`Modules URL: ${modulesTarUrl}`); - console.log(`Image URL: ${imageUrl}`); - console.log(`Vmlinux URL: ${vmlinuxUrl}`); - console.log(`Ramdisk FastRPC URL: ${firmwareUrl}`); - console.log(`Dtb URL: ${dtbUrl}`); - - - name: Create metadata.json - id: create_metadata - shell: bash - run: | - echo "Creating job dtb definition" - # Create the job definition using the processed URLs - cd ../job_render - docker run -i --rm \ - --user "$(id -u):$(id -g)" \ - --workdir="$PWD" \ - -v "$(dirname "$PWD")":"$(dirname "$PWD")" \ - -e dtb_url="${{ steps.process_urls.outputs.dtb_url }}" \ - ${{ inputs.docker_image }} \ - jq '.artifacts["dtbs/qcom/${{ env.DEVICE_TREE }}.dtb"] = env.dtb_url' data/metadata.json > temp.json && mv temp.json data/metadata.json - - - name: Upload metadata.json - id: upload_metadata - uses: qualcomm/fastrpc/.github/actions/aws_s3_helper@development - with: - local_file: ../job_render/data/metadata.json - s3_bucket: qli-prd-fastrpc-gh-artifacts - mode: single-upload - - - name: Create cloudData json - shell: bash - run: | - echo "Creating job metadata definition" - metadata_url="${{ steps.upload_metadata.outputs.presigned_url }}" - ramdisk_url="${{ steps.process_urls.outputs.ramdisk_url }}" - firmware_url="${{ steps.process_urls.outputs.firmware_url }}" - vmlinux_url="${{ steps.process_urls.outputs.vmlinux_url }}" - image_url="${{ steps.process_urls.outputs.image_url }}" - modules_url="${{ steps.process_urls.outputs.modules_url }}" - - # Create the job definition using the processed URLs - cd ../job_render - echo "Creating metadata_url ${metadata_url}" - # using metadata_url - docker run -i --rm \ - --user "$(id -u):$(id -g)" \ - --workdir="$PWD" \ - -v "$(dirname "$PWD")":"$(dirname "$PWD")" \ - -e metadata_url="$metadata_url" \ - ${{ inputs.docker_image }} \ - jq '.artifacts.metadata = env.metadata_url' data/cloudData.json > temp.json && mv temp.json data/cloudData.json - - echo "Creating image_url ${image_url}" - # using image_url - docker run -i --rm \ - --user "$(id -u):$(id -g)" \ - --workdir="$PWD" \ - -v "$(dirname "$PWD")":"$(dirname "$PWD")" \ - -e image_url="$image_url" \ - ${{ inputs.docker_image }} \ - jq '.artifacts.kernel = env.image_url' data/cloudData.json > temp.json && mv temp.json data/cloudData.json - - echo "Creating vmlinux_url ${vmlinux_url}" - # using vmlinux_url - docker run -i --rm \ - --user "$(id -u):$(id -g)" \ - --workdir="$PWD" \ - -v "$(dirname "$PWD")":"$(dirname "$PWD")" \ - -e vmlinux_url="$vmlinux_url" \ - ${{ inputs.docker_image }} \ - jq '.artifacts.vmlinux = env.vmlinux_url' data/cloudData.json > temp.json && mv temp.json data/cloudData.json - - echo "Creating FastRPC firmware_url ${firmware_url}" - # using firmware_url - ramdisk_fastrpc - docker run -i --rm \ - --user "$(id -u):$(id -g)" \ - --workdir="$PWD" \ - -v "$(dirname "$PWD")":"$(dirname "$PWD")" \ - -e firmware_url="$firmware_url" \ - ${{ inputs.docker_image }} \ - jq '.artifacts.firmware = env.firmware_url' data/cloudData.json > temp.json && mv temp.json data/cloudData.json - - echo "Creating modules_url ${modules_url}" - # using modules_url - docker run -i --rm \ - --user "$(id -u):$(id -g)" \ - --workdir="$PWD" \ - -v "$(dirname "$PWD")":"$(dirname "$PWD")" \ - -e modules_url="$modules_url" \ - ${{ inputs.docker_image }} \ - jq '.artifacts.modules = env.modules_url' data/cloudData.json > temp.json && mv temp.json data/cloudData.json - - - name: Update firmware to cloudData - shell: bash - run: | - cd ../job_render - ramdisk_url="$(aws s3 presign s3://qli-prd-fastrpc-gh-artifacts/meta-qcom/initramfs-kerneltest-full-image-qcom-armv8a.cpio.gz --expires 7600)" - echo "Updating ramdisk_url ${ramdisk_url}" - # using ramdisk_url - docker run -i --rm \ - --user "$(id -u):$(id -g)" \ - --workdir="$PWD" \ - -v "$(dirname "$PWD")":"$(dirname "$PWD")" \ - -e ramdisk_url="$ramdisk_url" \ - ${{ inputs.docker_image }} \ - jq '.artifacts.ramdisk = env.ramdisk_url' data/cloudData.json > temp.json && mv temp.json data/cloudData.json - - - name: Create lava_job_definition - shell: bash - run: | - cd ../job_render - mkdir -p renders - - docker run -i --rm \ - --user "$(id -u):$(id -g)" \ - --workdir="$PWD" \ - -v "$(dirname "$PWD")":"$(dirname "$PWD")" \ - -e TARGET="${{ env.LAVA_DEVICE_NAME }}" \ - -e TARGET_DTB="${{ env.DEVICE_TREE }}" \ - ${{ inputs.docker_image }} \ - sh -c 'export BOOT_METHOD=fastboot && \ - export TEST_METHOD=generic && \ - export TARGET=${TARGET} && \ - export TARGET_DTB=${TARGET_DTB} && \ - python3 lava_Job_definition_generator.py --localjson ./data/cloudData.json --fastrpc-premerge' diff --git a/.github/actions/loading/action.yml b/.github/actions/loading/action.yml deleted file mode 100644 index 5b82e0cb..00000000 --- a/.github/actions/loading/action.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Load Parameters -description: | - This composite action loads build parameters from a `MACHINES.json` file. - It parses the JSON file to create two matrices: - - `build_matrix`: A complete matrix containing 'deviceTree', 'linuxFirmware', 'lavaDeviceName' and 'hexDSPBinary' - It expects the `MACHINES.json` file to be located at `ci/MACHINES.json` relative to the workspace root. - -outputs: - build_matrix: - description: Build matrix - value: ${{ steps.set-matrix.outputs.build_matrix }} - -runs: - using: "composite" - steps: - - name: Set Build Matrix - id: set-matrix - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - const path = require('path'); - const filePath = path.join(process.env.GITHUB_WORKSPACE, 'ci', 'MACHINES.json'); - let fileData; - try { - if (!fs.existsSync(filePath)) { - core.setFailed(`Error: MACHINES.json not found at ${filePath}`); - return; - } - fileData = JSON.parse(fs.readFileSync(filePath, 'utf-8')); - } catch (err) { - core.setFailed(`Error loading or parsing MACHINES.json: ${err.message}`); - return; - } - - // fileData is an object, not an array - const matrix = Object.values(fileData) - .filter(item => - typeof item === 'object' && item !== null && - 'deviceTree' in item && 'linuxFirmware' in item && 'lavaDeviceName' in item && 'hexDSPBinary' in item - ) - .map(({ deviceTree, linuxFirmware, lavaDeviceName, hexDSPBinary }) => ({ - deviceTree, - linuxFirmware, - lavaDeviceName, - hexDSPBinary - })); - - core.setOutput('build_matrix', JSON.stringify(matrix)); - console.log("Generated build_matrix:"); - console.log(matrix); diff --git a/.github/actions/sync/action.yml b/.github/actions/sync/action.yml deleted file mode 100644 index 677ed748..00000000 --- a/.github/actions/sync/action.yml +++ /dev/null @@ -1,113 +0,0 @@ -name: Sync workspace -description: | - This composite action synchronizes a workspace by cleaning it - up,checking out necessary code repositories, and preparing a - linux Firmware directory with required binaries. - It clones and copies FastRPC hexagon DSP binaries and - Linux firmware (specifically QCOM firmware) based on - the provided `hexDSPBinary` and `linuxFirmware` input. - The action sets an output `workspace_path` to the GitHub - workspace directory. - -inputs: - linuxFirmware: - description: Linux Firmware identifier - type: string - required: true - hexDSPBinary: - description: Hexagon DSP binaries identifier - type: string - required: true - -outputs: - workspace_path: - description: Sync workspace path - value: ${{ steps.set-workspace.outputs.workspace }} - -runs: - using: "composite" - steps: - - name: Clean workspace - shell: bash - run: | - echo "Cleaning up workspace..." - rm -rf ${{ github.workspace }}/* - echo "Workspace cleaned successfully!" - - - name: Checkout fastrpc code - uses: actions/checkout@v4 - - - name: Clone FastRPC hexagon DSP binaries repositories and copy Firmware - shell: bash - run: | - cd ${{ github.workspace }} - mkdir -p firmware_dir/usr/lib/dsp/ - - echo "Cloning FastRPC hexagon DSP binaries..." - git clone https://github.com/linux-msm/hexagon-dsp-binaries.git - cd hexagon-dsp-binaries - git fetch --all --tags --prune - - TARGET="${{ inputs.hexDSPBinary }}" - DSP_REF="" - CONFIG="config.txt" - - echo "Searching FastRPC hexagon DSP binaries for target: $TARGET" - if [ ! -f "$CONFIG" ]; then - echo "Error: $CONFIG file not found in hexagon-dsp-binaries directory." - exit 1 - fi - while read -r _ subdir dsp version; do - echo "DSP binaries for $dsp in $subdir - $version" - # match anywhere in the path - if [[ "${subdir,,}" == *"${TARGET,,}/"* ]]; then - echo "Found matching DSP binaries for target '$TARGET': $subdir - $version" - echo "Copying DSP binaries for $dsp in $subdir/$version" - mkdir -p "../firmware_dir/usr/lib/dsp/${dsp}" - cp -rf "$subdir/$version/"* "../firmware_dir/usr/lib/dsp/${dsp}/" - echo "Copied DSP binaries for $dsp in ../firmware_dir/usr/lib/dsp/${dsp}" - DSP_REF="$version" - echo "${dsp}:${subdir}////${version}" - fi - done < <(grep "^Install:" "$CONFIG") - echo "DSP binaries copied successfully!" - - - name: Clone Firmware binaries repositories and copy Firmware - shell: bash - run: | - cd ${{ github.workspace }} - git clone https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git - TARGET="${{ inputs.hexDSPBinary }}" - LINUX_FIRMWARE="${{ inputs.linuxFirmware }}" - - # Construct the source path for qcom firmware in the cloned linux-firmware repo - # It's usually under /qcom/ - LINUX_FIRMWARE_QCOM_PATH="linux-firmware/qcom/${TARGET}" - - # Check if the target qcom firmware directory exists in the cloned repo - if [ ! -d "$LINUX_FIRMWARE_QCOM_PATH" ]; then - echo "ERROR: Directory '$LINUX_FIRMWARE_QCOM_PATH' not found in linux_firmware repo." - exit -1 - else - # Create the destination directory in firmware_dir - mkdir -p "${{ github.workspace }}/firmware_dir/usr/lib/firmware/qcom/${LINUX_FIRMWARE}" - - # Copy the contents of the qcom target directory - # Use trailing slashes to copy contents, not the directory itself - cp -rf "${LINUX_FIRMWARE_QCOM_PATH}/"* "${{ github.workspace }}/firmware_dir/usr/lib/firmware/qcom/${LINUX_FIRMWARE}/" - echo "Copied QCOM firmware from '${LINUX_FIRMWARE_QCOM_PATH}' to '${{ github.workspace }}/firmware_dir/usr/lib/firmware/qcom/${LINUX_FIRMWARE}/' successfully!" - fi - - - name: List all subdirectories and files in firmware_dir - shell: bash - run: | - cd ${{ github.workspace }} - echo "Listing contents of firmware_dir recursively..." - find firmware_dir -type d -print - find firmware_dir -type f -print - - - name: Set workspace path - id: set-workspace - shell: bash - run: | - echo "workspace=${{ github.workspace }}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/loading.yml b/.github/workflows/loading.yml deleted file mode 100644 index d4b04b3b..00000000 --- a/.github/workflows/loading.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: loading -description: Load required parameters for the subsequent jobs - -on: - workflow_call: - outputs: - build_matrix: - description: Build matrix - value: ${{ jobs.loading.outputs.build_matrix }} - -jobs: - loading: - runs-on: ubuntu-latest - outputs: - build_matrix: ${{ steps.loading.outputs.build_matrix }} - - steps: - - name: Checkout Code - uses: actions/checkout@v6 - - - name: Load Parameters - id: loading - uses: qualcomm/fastrpc/.github/actions/loading@development diff --git a/.github/workflows/nightly-kernel-build.yml b/.github/workflows/nightly-kernel-build.yml deleted file mode 100644 index 7eed74f5..00000000 --- a/.github/workflows/nightly-kernel-build.yml +++ /dev/null @@ -1,89 +0,0 @@ -# Nightly Build kernel and upload artifacts -# This workflow builds the Linux kernel nightly from the `qcom-next` branch -# of the `qualcomm-linux/kernel` repository using a custom Docker image. -# Artifacts are zipped and uploaded for 30 days retention. - -name: Nightly Build kernel and upload artifacts - -on: - schedule: - - cron: '0 0 * * 0' # Runs at 00:00, only on Sunday UTC - workflow_dispatch: # Allows manual trigger - -jobs: - build-and-upload: - runs-on: - group: GHA-fastrpc-Prd-SelfHosted-RG - labels: [ self-hosted, fastrpc-prd-u2404-x64-large-od-ephem ] - steps: - # Checkout fastrpc-image repository to get the Dockerfile - - name: Checkout fastrpc-image repo for Dockerfile - uses: actions/checkout@v6 - with: - fetch-depth: 0 - repository: qualcomm/fastrpc-image - ref: main - - - name: Build fastrpc docker image - run: | - docker build -t fastrpc-image:latest -f Dockerfile . - - - name: Configure git - shell: bash - run: | - git config --global user.name "github-actions" - git config --global user.email "github-actions@github.com" - - - name: Clone kernel repository - shell: bash - run: | - echo "Cloning qualcomm-linux/kernel into ${{ github.workspace }}/kernel..." - git clone https://github.com/qualcomm-linux/kernel.git "${{ github.workspace }}/kernel" - cd "${{ github.workspace }}/kernel" - echo "Fetching all branches and checking out 'qcom-next'..." - git fetch origin - git checkout qcom-next - echo "Successfully checked out qcom-next branch." - - - name: Build kernel - shell: bash - run: | - echo "Creating kobj directory for out-of-tree build output..." - mkdir -p "${{ github.workspace }}/kobj" - echo "Running Docker container to build the kernel..." - docker run -i --rm \ - --user "$(id -u):$(id -g)" \ - --volume "${{ github.workspace }}:/workspace" \ - --workdir="/workspace/kernel" \ - fastrpc-image:latest bash -c " - echo 'Inside container: Current working directory is $(pwd)' - echo 'Inside container: Building kernel with output directory O=/workspace/kobj' - make O=/workspace/kobj defconfig - make O=/workspace/kobj -j\$(nproc) all - make O=/workspace/kobj -j\$(nproc) dir-pkg INSTALL_MOD_STRIP=1 - " - echo "Kernel build completed. Checking for kobj output..." - ls -l "${{ github.workspace }}/kobj" || echo "kobj directory is empty or missing." - - - name: Zip kobj folder - shell: bash - run: | - mkdir -p build_output - if [ -d "${{ github.workspace }}/kobj" ]; then - echo "zipping kobj to build_output/kobj.tar.gz..." - tar -cvf build_output/kobj.tar -C "${{ github.workspace }}" kobj - gzip -9 build_output/kobj.tar - echo "Successfully created build_output/kobj.tar.gz" - else - echo "Error: kobj directory not found." - exit 1 - fi - - - name: Upload zipped kobj as artifact to aws S3 bucket - uses: qualcomm/fastrpc/.github/actions/aws_s3_helper@development - with: - s3_bucket: qli-prd-fastrpc-gh-artifacts - local_file: ${{ github.workspace }}/build_output/kobj.tar.gz - mode: single-upload - artifacts: kernel-artifact/kobj.tar.gz - deviceTree: false \ No newline at end of file diff --git a/.github/workflows/pre_merge.yml b/.github/workflows/pre_merge.yml deleted file mode 100644 index 55908d5b..00000000 --- a/.github/workflows/pre_merge.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: pre_merge -description: | - This workflow defines the pre_merge CI process for the fastrpc repository. - It is triggered on pushes to 'main' or 'development' branches, - on pull request events (opened, synchronize, reopened) targeting 'main' or 'development', - or manually via workflow_dispatch. - Loading: Loads the build matrix from MACHINES.json. - Build: Syncs and Builds code for each target using sync_build.yml. - Test: Runs LAVA tests using built artifacts via test.yml. - All jobs use reusable workflows and inherit secrets from the caller. - -on: - push: - branches: - - 'main' - - 'development' - workflow_dispatch: - pull_request: # requiring access to base repo - types: [opened, synchronize, reopened] - branches: - - 'main' - - 'development' - paths-ignore: - - '.github/**' - -jobs: - - # The loading job will load the build matrix from MACHINES.json - loading: - uses: qualcomm/fastrpc/.github/workflows/loading.yml@development - secrets: inherit - - # The build job will run on the machines specified in the build_matrix output - # from the loading job. It will use the build.yml workflow to build the code. - build: - needs: loading - strategy: - matrix: - include: ${{ fromJSON(needs.loading.outputs.build_matrix) }} - uses: qualcomm/fastrpc/.github/workflows/sync_build.yml@development - secrets: inherit - with: - docker_image: fastrpc-image:latest - deviceTree: ${{ matrix.deviceTree }} - linuxFirmware: ${{ matrix.linuxFirmware }} - hexDSPBinary: ${{ matrix.hexDSPBinary }} - - # The test job will run on the machines specified in the build_matrix output - # from the loading job. It will use the test.yml workflow to run tests. - test: - needs: [loading, build] - uses: qualcomm/fastrpc/.github/workflows/test.yml@development - secrets: inherit - with: - docker_image: fastrpc-image:latest - build_matrix: ${{ needs.loading.outputs.build_matrix }} diff --git a/.github/workflows/sync_build.yml b/.github/workflows/sync_build.yml deleted file mode 100644 index a483d888..00000000 --- a/.github/workflows/sync_build.yml +++ /dev/null @@ -1,117 +0,0 @@ -name: Sync and build -description: | - This reusable workflow syncs and builds FastRPC code for a specified machine and firmware - using a Docker image. It performs code checkout, Docker image build, workspace sync, - kernel compilation, and artifact packaging. Key build outputs (Image, ramdisk, modules, DTB) - are uploaded to S3, and the runner workspace is cleaned to manage disk usage. A summary of - the build status is added to the GitHub Actions summary. - -on: - workflow_call: - inputs: - docker_image: - description: Docker image - type: string - required: true - deviceTree: - description: device Tree name - type: string - required: true - linuxFirmware: - description: Linux Firmware identifier - type: string - required: true - hexDSPBinary: - description: Hexagon DSP binaries identifier - type: string - required: true - -jobs: - build: - runs-on: - group: GHA-fastrpc-Prd-SelfHosted-RG - labels: [ self-hosted, fastrpc-prd-u2404-x64-large-od-ephem ] - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - # fetch-depth: 1 is generally sufficient for builds and is faster than 0 (full history). - # Change to 0 if your build process strictly requires the full Git history. - fetch-depth: 1 - - - name: Build fastrpc docker image - uses: qualcomm/fastrpc/.github/actions/build_docker_image@development - with: - image: ${{ inputs.docker_image }} - - - name: Sync codebase - id: sync - uses: qualcomm/fastrpc/.github/actions/sync@development - with: - linuxFirmware: ${{ inputs.linuxFirmware }} - hexDSPBinary: ${{ inputs.hexDSPBinary }} - - - name: Build workspace - id: build_workspace - uses: qualcomm/fastrpc/.github/actions/build@development - with: - docker_image: ${{ inputs.docker_image }} - workspace_path: ${{ steps.sync.outputs.workspace_path }} - deviceTree: ${{ inputs.deviceTree }} - - - name: Create file list for artifacts upload - run: | - workspace=${{ steps.sync.outputs.workspace_path }} - - # Create the tarball in the root of the GitHub workspace - cd "$workspace/kobj/tar-install" - tar -cJf "${{ github.workspace }}/modules.tar.xz" lib/modules/ - - # Populate the file_list.txt with paths to artifacts - cat < "$workspace/artifacts/file_list.txt" - ${{ github.workspace }}/modules.tar.xz - $workspace/kobj/arch/arm64/boot/Image - $workspace/kobj/vmlinux - $workspace/artifacts/ramdisk_fastrpc.gz - $workspace/kobj/arch/arm64/boot/dts/qcom/${{ inputs.deviceTree }}.dtb - EOF - - echo "Generated artifact list:" - cat "$workspace/artifacts/file_list.txt" - - - name: Upload artifacts - uses: qualcomm/fastrpc/.github/actions/aws_s3_helper@development - with: - s3_bucket: qli-prd-fastrpc-gh-artifacts - # local_file points to the list of files to upload - local_file: ${{ steps.sync.outputs.workspace_path }}/artifacts/file_list.txt - mode: multi-upload - deviceTree: ${{ inputs.deviceTree }} - - - name: Clean up - # This step is crucial for self-hosted runners to manage disk space. - run: | - rm -rf "${{ steps.sync.outputs.workspace_path }}/artifacts" || true - rm -rf "${{ steps.sync.outputs.workspace_path }}/fastrpc_libs" || true - rm -rf "${{ steps.sync.outputs.workspace_path }}/gcc-linaro-7.5.0-2019.12-i686_aarch64-linux-gnu" || true - rm -rf "${{ steps.sync.outputs.workspace_path }}/kobj" || true - rm -f "${{ github.workspace }}/modules.tar.xz" || true # Clean up the tarball from the root workspace - - - name: Update summary - # This step always runs to provide feedback on the build status. - if: success() || failure() - shell: bash - run: | - if [ "${{ steps.build_workspace.outcome }}" == 'success' ]; then - echo "Build was successful" - summary=":heavy_check_mark: Build Success" - else - echo "Build failed" - summary=":x: Build Failed" - fi - SUMMARY=' -
Build Summary - '${summary}' -
- ' - echo -e "$SUMMARY" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 28b020d0..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,135 +0,0 @@ -name: test -description: | - Run tests on LAVA for a specified target using a Docker image. - This reusable GitHub Actions workflow executes FastRPC tests on LAVA infrastructure. - It leverages a Docker image for test execution and supports multi-target builds via a matrix strategy. - The workflow automates job creation, submission, and result validation on LAVA, - providing a streamlined testing process. - Designed to be triggered by other workflows, it offers a summary of test outcomes - with direct links to LAVA job results, enhancing traceability and debugging. - -on: - workflow_call: - inputs: - docker_image: - description: Docker image - type: string - required: true - default: fastrpc-image:latest - - build_matrix: - description: Build matrix for multi target builds - type: string - required: true - -jobs: - test: - runs-on: - group: GHA-fastrpc-Prd-SelfHosted-RG - labels: [ self-hosted, fastrpc-prd-u2404-x64-large-od-ephem ] - strategy: - fail-fast: false - matrix: - build_matrix: ${{ fromJson(inputs.build_matrix) }} - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - ref: ${{ github.ref }} - fetch-depth: 0 - - - name: Build fastrpc docker image - uses: qualcomm/fastrpc/.github/actions/build_docker_image@development - with: - image: ${{ inputs.docker_image }} - - - name: Download URLs list - uses: actions/download-artifact@v8 - with: - name: presigned_urls_${{ matrix.build_matrix.deviceTree }}.json - merge-multiple: true - path: ${{ github.workspace }} - - - name: Clone lava job render scripts - run: cd .. && git clone https://github.com/qualcomm-linux/job_render - - - name: Create fastrpc - lava job definition - uses: qualcomm/fastrpc/.github/actions/lava_job_render@development - id: create_job_definition - with: - docker_image: ${{ inputs.docker_image }} - env: - FIRMWARE: ${{ matrix.build_matrix.linuxFirmware }} - DEVICE_TREE: ${{ matrix.build_matrix.deviceTree }} - LAVA_DEVICE_NAME: ${{ matrix.build_matrix.lavaDeviceName }} - - - name: Submit lava job - id: submit_job - run: | - cd ../job_render - job_id=$(docker run -i --rm --workdir="$PWD" -v "$(dirname $PWD)":"$(dirname $PWD)" ${{ inputs.docker_image }} sh -c "lavacli identities add --token ${{ secrets.LAVA_OSS_TOKEN }} --uri https://lava-oss.qualcomm.com/RPC2 --username ${{ secrets.LAVA_OSS_USER }} production && lavacli -i production jobs submit ./renders/lava_job_definition.yaml") - job_url="https://lava-oss.qualcomm.com/scheduler/job/$job_id" - echo "job_id=$job_id" >> $GITHUB_OUTPUT - echo "job_url=$job_url" >> $GITHUB_OUTPUT - echo "Lava Job: $job_url" - echo "JOB_ID=$job_id" >> $GITHUB_ENV - - - name: Check lava job results - id: check_job - run: | - STATE="" - START_TIME=$(date +%s) - while [ "$STATE" != "Finished" ]; do - state=$(docker run -i --rm --workdir="$PWD" -v "$(dirname $PWD)":"$(dirname $PWD)" ${{ inputs.docker_image }} sh -c "lavacli identities add --token ${{ secrets.LAVA_OSS_TOKEN }} --uri https://lava-oss.qualcomm.com/RPC2 --username ${{ secrets.LAVA_OSS_USER }} production && lavacli -i production jobs show $JOB_ID" | grep state) - STATE=$(echo "$state" | cut -d':' -f2 | sed 's/^ *//;s/ *$//') - echo "Current status: $STATE" - CURRENT_TIME=$(date +%s) - ELAPSED_TIME=$(((CURRENT_TIME - START_TIME)/3600)) - if [ $ELAPSED_TIME -ge 2 ]; then - echo "Timeout: 2 hours exceeded." - summary=":x: Lava job exceeded time limit." - echo "summary=$summary" >> $GITHUB_OUTPUT - exit 1 - fi - sleep 30 - done - health=$(docker run -i --rm --workdir="$PWD" -v "$(dirname $PWD)":"$(dirname $PWD)" ${{ inputs.docker_image }} sh -c "lavacli identities add --token ${{ secrets.LAVA_OSS_TOKEN }} --uri https://lava-oss.qualcomm.com/RPC2 --username ${{ secrets.LAVA_OSS_USER }} production && lavacli -i production jobs show $JOB_ID" | grep Health) - HEALTH=$(echo "$health" | cut -d':' -f2 | sed 's/^ *//;s/ *$//') - if [[ "$HEALTH" == "Complete" ]]; then - TEST_RESULTS=$(docker run -i --rm --workdir="$PWD" -v "$(dirname $PWD)":"$(dirname $PWD)" ${{ inputs.docker_image }} sh -c "lavacli identities add --token ${{ secrets.LAVA_OSS_TOKEN }} --uri https://lava-oss.qualcomm.com/RPC2 --username ${{ secrets.LAVA_OSS_USER }} production && lavacli -i production results $JOB_ID" | grep fail || echo "Pass") - if [[ "$TEST_RESULTS" == "Pass" ]]; then - echo "Lava job passed." - summary=":heavy_check_mark: Lava job passed." - echo "summary=$summary" >> $GITHUB_OUTPUT - exit 0 - else - echo "Lava job failed." - summary=":x: Lava job failed." - echo "summary=$summary" >> $GITHUB_OUTPUT - exit 1 - fi - else - echo "Lava job failed." - summary=":x: Lava job failed." - echo "summary=$summary" >> $GITHUB_OUTPUT - exit 1 - fi - - - name: Update summary - if: success() || failure() - shell: bash - run: | - if [ "${{ steps.create_job_definition.conclusion }}" == 'failure' ]; then - status=":x: Test job failed" - else - status="${{ steps.check_job.outputs.summary }}" - job_url="${{ steps.submit_job.outputs.job_url }}" - job_id="${{ steps.submit_job.outputs.job_id }}" - fi - SUMMARY=' -
'${status}' -
- JOB ID: '${job_id}' -
- ' - echo -e "$SUMMARY" >> $GITHUB_STEP_SUMMARY diff --git a/ci/MACHINES.json b/ci/MACHINES.json deleted file mode 100644 index df35c5a9..00000000 --- a/ci/MACHINES.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "qcs6490-rb3gen2": { - "deviceTree": "qcs6490-rb3gen2", - "linuxFirmware": "qcs6490", - "lavaDeviceName": "qcs6490", - "hexDSPBinary": "qcm6490" - }, - "qcs9100-ride-r3": { - "deviceTree": "qcs9100-ride-r3", - "linuxFirmware": "sa8775p", - "lavaDeviceName": "qcs9100-ride", - "hexDSPBinary": "sa8775p" - }, - "qcs8300-ride": { - "deviceTree": "qcs8300-ride", - "linuxFirmware": "qcs8300", - "lavaDeviceName": "qcs8300-ride", - "hexDSPBinary": "qcs8300" - }, - "qcs615-ride": { - "deviceTree": "qcs615-ride", - "linuxFirmware": "qcs615", - "lavaDeviceName": "qcs615-ride", - "hexDSPBinary": "qcs615" - } -} \ No newline at end of file