diff --git a/Jenkinsfile b/Jenkinsfile index f0e6b3dcf5ef1e..d9e5d599e94bc0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,3 +1,11 @@ +def ciDockerImage() { + return 'ghcr.io/commaai/alpine-ssh@sha256:e79076caaa7e8bc766fd07e81fe0db5d0449bee936d21a1c11dbe4b95a412063' +} + +def ciSyncEpoch() { + return 'manifest-after-content-v2' +} + def retryWithDelay(int maxRetries, int delay, Closure body) { for (int i = 0; i < maxRetries; i++) { try { @@ -9,10 +17,10 @@ def retryWithDelay(int maxRetries, int delay, Closure body) { throw Exception("Failed after ${maxRetries} retries") } -def device(String ip, String step_label, String cmd) { +def device(String ip, String step_label, String cmd, String controlPath = "") { withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) { def ssh_cmd = """ -ssh -o ConnectTimeout=5 -o ServerAliveInterval=5 -o ServerAliveCountMax=2 -o BatchMode=yes -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' exec /usr/bin/bash <<'END' +${rsyncSshCommand(key_file, controlPath)} 'comma@${ip}' exec /usr/bin/bash <<'END' set -e @@ -73,6 +81,374 @@ END""" } } +def ciDockerArgs() { + return '--user=root -v /var/jenkins_home/openpilot-device-build-cache:/device-build-cache' +} + +def cleanOldWorkspaceBuildCache() { + docker.image(ciDockerImage()).inside(ciDockerArgs()) { + sh script: "rm -rf '${env.WORKSPACE}/.device-build-cache'", label: 'clean old workspace build cache' + } +} + +def rsyncSshCommand(String keyFile, String controlPath = "") { + def controlArgs = controlPath ? " -o ControlMaster=no -o ControlPath=${controlPath}" : "" + return "ssh -o ConnectTimeout=5 -o ServerAliveInterval=5 -o ServerAliveCountMax=2 -o BatchMode=yes -o StrictHostKeyChecking=no -i ${keyFile}${controlArgs}" +} + +def sshControlPath(String ip) { + def buildId = (env.BUILD_NUMBER ?: 'local').replaceAll('[^A-Za-z0-9_.-]', '_') + return "/tmp/op-ci-ssh-${ip.replaceAll('[^A-Za-z0-9_.-]', '_')}-${buildId}" +} + +def startSshMaster(String ip, String controlPath) { + withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) { + sh script: """ +set -e +rm -f '${controlPath}' +${rsyncSshCommand(key_file)} -o ControlMaster=yes -o ControlPath='${controlPath}' -o ControlPersist=30s 'comma@${ip}' -Nf +""", label: 'start ssh master' + } +} + +def rsyncBuiltTreeFromDevice(String ip, String controlPath = "") { + withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) { + sh script: """ +set -e + +cache='${env.DEVICE_BUILD_DIR}' +mkdir -p "\${cache}" +rm -rf "\${cache}/.git" "\${cache}/.venv" "\${cache}/.ruff_cache" "\${cache}/.pytest_cache" "\${cache}/.mypy_cache" +rm -f "\${cache}/.sconsign.dblite" + +old_manifest="\${cache}/.ci_manifest" +old_id="\$(cat "\${cache}/.ci_manifest.id" 2>/dev/null || true)" +manifest_dir="\${cache}/.ci_manifests" +mkdir -p "\${manifest_dir}" +if [ -n "\${old_id}" ] && [ -f "\${old_manifest}" ]; then + cp "\${old_manifest}" "\${manifest_dir}/\${old_id}" +fi + +new_manifest="\$(mktemp)" +new_id_file="\$(mktemp)" +sync_paths="\${cache}/.ci_sync_paths" +changed_paths="\${cache}/.ci_changed_paths" +deleted_paths="\${cache}/.ci_deleted_paths" +cache_updated=0 +trap 'rm -f "\${new_manifest}" "\${new_id_file}"' EXIT + +ssh_cmd='${rsyncSshCommand(key_file, controlPath)}' +ssh_cmd="\${ssh_cmd} comma@${ip}" + +\${ssh_cmd} "cat '${env.TEST_DIR}/.ci_manifest.id'" > "\${new_id_file}" +new_id="\$(cat "\${new_id_file}")" +printf '%s\\n' "\${old_id}" > "\${cache}/.ci_previous_manifest.id" + +if [ -f "\${old_manifest}" ] && [ "\${old_id}" = "\${new_id}" ]; then + : > "\${changed_paths}" + : > "\${deleted_paths}" + : > "\${sync_paths}" + echo "builder cache manifest unchanged: \${new_id}" +elif [ -f "\${old_manifest}" ]; then + \${ssh_cmd} "cat '${env.TEST_DIR}/.ci_manifest'" > "\${new_manifest}" + old_entries="\$(mktemp)" + new_entries="\$(mktemp)" + old_paths="\$(mktemp)" + new_paths="\$(mktemp)" + trap 'rm -f "\${new_manifest}" "\${new_id_file}" "\${old_entries}" "\${new_entries}" "\${old_paths}" "\${new_paths}"' EXIT + awk -F '\\t' 'BEGIN { OFS = "\\t" } { print \$3, \$1, \$2 }' "\${old_manifest}" | sort > "\${old_entries}" + awk -F '\\t' 'BEGIN { OFS = "\\t" } { print \$3, \$1, \$2 }' "\${new_manifest}" | sort > "\${new_entries}" + cut -f1 "\${old_entries}" > "\${old_paths}" + cut -f1 "\${new_entries}" > "\${new_paths}" + comm -13 "\${old_entries}" "\${new_entries}" | cut -f1 > "\${changed_paths}" + comm -23 "\${old_paths}" "\${new_paths}" > "\${deleted_paths}" + cat "\${changed_paths}" > "\${sync_paths}" + echo "changed=\$(wc -l < "\${changed_paths}") deleted=\$(wc -l < "\${deleted_paths}")" + sed -n '1,40p' "\${changed_paths}" + if [ -s "\${deleted_paths}" ]; then + while IFS= read -r path; do + [ -n "\${path}" ] || continue + case "\${path}" in + /*|../*|*/../*) echo "refusing to delete unexpected path \${path}" >&2; exit 1 ;; + esac + rm -rf -- "\${cache}/\${path}" + done < "\${deleted_paths}" + fi + rsync -a --delete-missing-args --no-owner --no-group --info=stats2,name0 \\ + --ignore-times \\ + --files-from="\${sync_paths}" \\ + -e '${rsyncSshCommand(key_file, controlPath)}' \\ + 'comma@${ip}:${env.TEST_DIR}/' "\${cache}/" + cache_updated=1 +else + \${ssh_cmd} "cat '${env.TEST_DIR}/.ci_manifest'" > "\${new_manifest}" + echo "builder cache has no manifest, doing full content sync" + rsync -a --delete --delete-excluded --checksum --no-owner --no-group --info=stats2,name0 \\ + --exclude='.git' --exclude='.git/' --exclude='.git/**' \\ + --exclude='.venv' --exclude='.ruff_cache' --exclude='.pytest_cache' --exclude='.mypy_cache' \\ + --exclude='__pycache__' --exclude='.sconsign.dblite' \\ + --exclude='.ci_manifest' --exclude='.ci_manifest.id' --exclude='.ci_sync_epoch' \\ + -e '${rsyncSshCommand(key_file, controlPath)}' \\ + 'comma@${ip}:${env.TEST_DIR}/' "\${cache}/" + find "\${cache}" -depth -type d -empty -delete + : > "\${sync_paths}" + : > "\${changed_paths}" + : > "\${deleted_paths}" + cache_updated=1 +fi + +if [ "\${cache_updated}" = "1" ]; then + cp "\${new_manifest}" "\${cache}/.ci_manifest" + cp "\${new_id_file}" "\${cache}/.ci_manifest.id" + cp "\${new_manifest}" "\${manifest_dir}/\${new_id}" +else + cp "\${old_manifest}" "\${manifest_dir}/\${new_id}" +fi +printf '%s\\n' '${ciSyncEpoch()}' > "\${cache}/.ci_sync_epoch" +printf '%s\\n' "\${new_id}" > "\${cache}/.ci_current_manifest.id" +echo "builder cache previous manifest: \${old_id:-none}" +echo "builder cache current manifest: \${new_id}" +echo "builder cache changed paths: \$(wc -l < "\${changed_paths}")" +echo "builder cache deleted paths: \$(wc -l < "\${deleted_paths}")" +""", label: 'cache built tree' + } + + env.DEVICE_BUILD_PREVIOUS_ID = sh(script: "cat '${env.DEVICE_BUILD_DIR}/.ci_previous_manifest.id' 2>/dev/null || true", returnStdout: true).trim() + env.DEVICE_BUILD_ID = sh(script: "cat '${env.DEVICE_BUILD_DIR}/.ci_current_manifest.id'", returnStdout: true).trim() +} + +def rsyncBuiltTreeToDevice(String ip, String controlPath = "") { + if (env.DEVICE_BUILD_READY != "1") { + return + } + + withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) { + sh script: """ +set -e + +cache='${env.DEVICE_BUILD_DIR}' +previous_id='${env.DEVICE_BUILD_PREVIOUS_ID ?: ''}' +current_id='${env.DEVICE_BUILD_ID ?: ''}' +sync_epoch='${ciSyncEpoch()}' +ssh_cmd='${rsyncSshCommand(key_file, controlPath)}' +remote="comma@${ip}" +remote_manifest="" + +sync_manifest_to_device() { + manifest_paths="\$(mktemp)" + printf '.ci_manifest\\n.ci_manifest.id\\n' > "\${manifest_paths}" + rsync -a --no-owner --no-group --info=stats2,name0 \\ + --ignore-times \\ + --files-from="\${manifest_paths}" \\ + -e '${rsyncSshCommand(key_file, controlPath)}' \\ + "\${cache}/" "\${remote}:${env.TEST_DIR}/" + rm -f "\${manifest_paths}" + \${ssh_cmd} "\${remote}" "printf '%s\\n' '\${sync_epoch}' > '${env.TEST_DIR}/.ci_sync_epoch'" +} + +remote_id="\$(\${ssh_cmd} "\${remote}" "cat '${env.TEST_DIR}/.ci_manifest.id' 2>/dev/null || true")" +remote_epoch="\$(\${ssh_cmd} "\${remote}" "cat '${env.TEST_DIR}/.ci_sync_epoch' 2>/dev/null || true")" +if [ "\${remote_epoch}" != "\${sync_epoch}" ]; then + echo "device sync epoch was \${remote_epoch:-none}, forcing full content sync" + remote_id="" +fi +if [ -n "\${remote_id}" ] && [ -f "\${cache}/.ci_manifests/\${remote_id}" ]; then + remote_manifest="\${cache}/.ci_manifests/\${remote_id}" +fi + +if [ -n "\${current_id}" ] && [ "\${remote_id}" = "\${current_id}" ]; then + echo "device already has manifest \${current_id}" +elif [ -n "\${remote_id}" ] && [ -f "\${remote_manifest}" ]; then + echo "delta sync from \${remote_id} to \${current_id}" + \${ssh_cmd} "\${remote}" "mkdir -p '${env.TEST_DIR}' && rm -rf '${env.TEST_DIR}/.git' '${env.TEST_DIR}/.venv' '${env.TEST_DIR}/.ruff_cache' '${env.TEST_DIR}/.pytest_cache' '${env.TEST_DIR}/.mypy_cache' && rm -f '${env.TEST_DIR}/.sconsign.dblite'" + current_manifest="\${cache}/.ci_manifest" + old_entries="\$(mktemp)" + new_entries="\$(mktemp)" + old_paths="\$(mktemp)" + new_paths="\$(mktemp)" + sync_paths="\$(mktemp)" + changed_paths="\$(mktemp)" + deleted_paths="\$(mktemp)" + trap 'rm -f "\${old_entries}" "\${new_entries}" "\${old_paths}" "\${new_paths}" "\${sync_paths}" "\${changed_paths}" "\${deleted_paths}"' EXIT + awk -F '\\t' 'BEGIN { OFS = "\\t" } { print \$3, \$1, \$2 }' "\${remote_manifest}" | sort > "\${old_entries}" + awk -F '\\t' 'BEGIN { OFS = "\\t" } { print \$3, \$1, \$2 }' "\${current_manifest}" | sort > "\${new_entries}" + cut -f1 "\${old_entries}" > "\${old_paths}" + cut -f1 "\${new_entries}" > "\${new_paths}" + comm -13 "\${old_entries}" "\${new_entries}" | cut -f1 > "\${changed_paths}" + comm -23 "\${old_paths}" "\${new_paths}" > "\${deleted_paths}" + cat "\${changed_paths}" > "\${sync_paths}" + echo "changed=\$(wc -l < "\${changed_paths}") deleted=\$(wc -l < "\${deleted_paths}")" + sed -n '1,40p' "\${changed_paths}" + if [ -s "\${deleted_paths}" ]; then + \${ssh_cmd} "\${remote}" "cd '${env.TEST_DIR}' && while IFS= read -r path; do [ -n \"\$path\" ] || continue; case \"\$path\" in /*|../*|*/../*) echo \"refusing to delete unexpected path \$path\" >&2; exit 1 ;; esac; rm -rf -- \"\$path\"; done" < "\${deleted_paths}" + fi + if [ -s "\${sync_paths}" ]; then + rsync -a --delete-missing-args --no-owner --no-group --info=stats2,name0 \\ + --ignore-times \\ + --files-from="\${sync_paths}" \\ + -e '${rsyncSshCommand(key_file, controlPath)}' \\ + "\${cache}/" "\${remote}:${env.TEST_DIR}/" + else + echo "no changed paths to sync" + fi + sync_manifest_to_device +else + echo "full content sync, remote manifest was \${remote_id:-none}, expected \${previous_id:-none}" + \${ssh_cmd} "\${remote}" "mkdir -p '${env.TEST_DIR}' && rm -rf '${env.TEST_DIR}/.git' '${env.TEST_DIR}/.venv' '${env.TEST_DIR}/.ruff_cache' '${env.TEST_DIR}/.pytest_cache' '${env.TEST_DIR}/.mypy_cache' && rm -f '${env.TEST_DIR}/.sconsign.dblite'" + rsync -a --delete --delete-excluded --checksum --no-owner --no-group --info=stats2,name0 \\ + --exclude='.git' --exclude='.git/' --exclude='.git/**' \\ + --exclude='.venv' --exclude='.ruff_cache' --exclude='.pytest_cache' --exclude='.mypy_cache' \\ + --exclude='__pycache__' --exclude='.sconsign.dblite' \\ + --exclude='.ci_manifest' --exclude='.ci_manifest.id' --exclude='.ci_sync_epoch' \\ + --exclude='.ci_changed_paths' --exclude='.ci_deleted_paths' --exclude='.ci_sync_paths' \\ + --exclude='.ci_previous_manifest.id' --exclude='.ci_current_manifest.id' \\ + -e '${rsyncSshCommand(key_file, controlPath)}' \\ + "\${cache}/" "\${remote}:${env.TEST_DIR}/" + sync_manifest_to_device +fi +""", label: 'sync built tree' + } +} + +def buildAndPrepareTreeCommand() { + return """ +if [ ! -f /tmp/openpilot_ci_cpu_unlocked ]; then + for cpu in /sys/devices/system/cpu/cpu[0-9]*; do + online="\${cpu}/online" + if [ -w "\${online}" ]; then + echo 1 | sudo tee "\${online}" + fi + done + + for policy in /sys/devices/system/cpu/cpufreq/policy*; do + [ -d "\${policy}" ] || continue + max="\$(cat "\${policy}/cpuinfo_max_freq")" + echo "\${max}" | sudo tee "\${policy}/scaling_max_freq" + echo "\${max}" | sudo tee "\${policy}/scaling_min_freq" + if grep -qw performance "\${policy}/scaling_available_governors"; then + echo performance | sudo tee "\${policy}/scaling_governor" + fi + done + + for governor in /sys/class/devfreq/soc:qcom,cpubw/governor /sys/class/devfreq/soc:qcom,memlat-cpu*/governor; do + if [ -w "\${governor}" ]; then + echo performance | sudo tee "\${governor}" + fi + done + + touch /tmp/openpilot_ci_cpu_unlocked +fi + +cd system/manager +taskset -c 0-7 ./build.py + +cd "\${TEST_DIR}" +dirty="\$(git status --porcelain)" +if [ -n "\${dirty}" ]; then + echo "Dirty working tree after build:" + echo "\${dirty}" + exit 1 +fi + +python3 - <<'PY' +import hashlib +import json +import os +import subprocess +from pathlib import Path + +root = Path(".") + +def run(cmd): + return subprocess.check_output(cmd, text=True).strip() + +metadata = { + "channel": "${env.BRANCH_NAME ?: 'unknown'}", + "openpilot": { + "version": (root / "common/version.h").read_text().split('"')[1], + "release_notes": (root / "RELEASES.md").read_text().split("\\n\\n", 1)[0], + "git_commit": run(["git", "rev-parse", "HEAD"]), + "git_origin": run(["git", "config", "--get", "remote.origin.url"]), + "git_commit_date": run(["git", "show", "--no-patch", "--format=%ct %ci", "HEAD"]), + "build_style": "ci", + }, +} +(root / "build.json").write_text(json.dumps(metadata)) + +raw_tracked = subprocess.check_output(["git", "ls-files", "--recurse-submodules", "-s", "-z"]) +tracked = {} +for record in raw_tracked.split(b"\\0"): + if not record: + continue + metadata, path = record.split(b"\\t", 1) + mode, oid, _stage = metadata.split(b" ") + tracked[path.decode()] = (mode.decode(), oid.decode()) + +entries = [("G", f"{mode}:{oid}", path) for path, (mode, oid) in tracked.items()] +skip_dirs = {".git", ".venv", ".ruff_cache", ".pytest_cache", ".mypy_cache", "__pycache__"} +skip_files = {".ci_manifest", ".ci_manifest.id", ".ci_sync_epoch", ".sconsign.dblite"} + +for dirpath, dirnames, filenames in os.walk(root): + dirpath = Path(dirpath) + + keep_dirs = [] + for name in sorted(dirnames): + path = dirpath / name + rel = path.relative_to(root).as_posix() + if name in skip_dirs or rel.startswith(".git/"): + continue + if rel in tracked: + continue + if path.is_symlink(): + entries.append(("L", os.readlink(path), rel)) + else: + keep_dirs.append(name) + dirnames[:] = keep_dirs + + for name in sorted(filenames): + path = dirpath / name + rel = path.relative_to(root).as_posix() + if rel in tracked or name == ".git" or name in skip_files: + continue + if path.is_symlink(): + entries.append(("L", os.readlink(path), rel)) + continue + + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + entries.append(("F", f"{path.stat().st_size}:{h.hexdigest()}", rel)) + +manifest = "".join(f"{kind}\\t{value}\\t{rel}\\n" for kind, value, rel in sorted(entries, key=lambda e: e[2])) +(root / ".ci_manifest").write_text(manifest) +(root / ".ci_manifest.id").write_text(hashlib.sha256(manifest.encode()).hexdigest() + "\\n") +print((root / "build.json").read_text()) +print(f"manifest entries={len(entries)} id={(root / '.ci_manifest.id').read_text().strip()}") +PY +""" +} + +def prepareBuiltTree() { + stage("build device tree") { + lock(resource: "comma-db5a74d4", inversePrecedence: true, variable: 'builder_ip') { + docker.image(ciDockerImage()).inside(ciDockerArgs()) { + timeout(time: 35, unit: 'MINUTES') { + def controlPath = sshControlPath(builder_ip) + startSshMaster(builder_ip, controlPath) + retry(3) { + def date = sh(script: 'date', returnStdout: true).trim(); + device(builder_ip, "builder git checkout", "date -s '" + date + "'\nexport UNSAFE=1\n" + readFile("selfdrive/test/setup_device_ci.sh"), controlPath) + } + device(builder_ip, "build and prepare tree", buildAndPrepareTreeCommand(), controlPath) + rsyncBuiltTreeFromDevice(builder_ip, controlPath) + env.DEVICE_BUILD_READY = "1" + } + } + } + } +} + def deviceStage(String stageName, String deviceType, List extra_env, def steps) { stage(stageName) { if (currentBuild.result != null) { @@ -88,12 +464,18 @@ def deviceStage(String stageName, String deviceType, List extra_env, def steps) def gitDiff = sh returnStdout: true, script: 'curl -s -H "Authorization: Bearer ${GITHUB_COMMENTS_TOKEN}" https://api.github.com/repos/commaai/openpilot/compare/master...${GIT_BRANCH} | jq .files[].filename || echo "/"', label: 'Getting changes' lock(resource: "", label: deviceType, inversePrecedence: true, variable: 'device_ip', quantity: 1, resourceSelectStrategy: 'random') { - docker.image('ghcr.io/commaai/alpine-ssh').inside('--user=root') { + docker.image(ciDockerImage()).inside(ciDockerArgs()) { timeout(time: 35, unit: 'MINUTES') { + def controlPath = sshControlPath(device_ip) + startSshMaster(device_ip, controlPath) retry (3) { def date = sh(script: 'date', returnStdout: true).trim(); - device(device_ip, "set time", "date -s '" + date + "'") - device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh")) + if (env.DEVICE_BUILD_READY == "1") { + device(device_ip, "device setup", "date -s '" + date + "'\nmkdir -p ${env.TEST_DIR}\nexport SKIP_GIT_CHECKOUT=1\n" + readFile("selfdrive/test/setup_device_ci.sh"), controlPath) + rsyncBuiltTreeToDevice(device_ip, controlPath) + } else { + device(device_ip, "git checkout", "date -s '" + date + "'\n" + extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh"), controlPath) + } } steps.each { item -> def name = item[0] @@ -103,12 +485,15 @@ def deviceStage(String stageName, String deviceType, List extra_env, def steps) def diffPaths = args.diffPaths ?: [] def cmdTimeout = args.timeout ?: 9999 - if (branch != "master" && !branch.contains("__jenkins_loop_") && diffPaths && !hasPathChanged(gitDiff, diffPaths)) { + if (env.DEVICE_BUILD_READY == "1" && (name.toLowerCase().startsWith("build") || name == "check dirty")) { + println "Skipping ${name}: using shared built tree." + return + } else if (branch != "master" && !branch.contains("__jenkins_loop_") && diffPaths && !hasPathChanged(gitDiff, diffPaths)) { println "Skipping ${name}: no changes in ${diffPaths}." return } else { timeout(time: cmdTimeout, unit: 'SECONDS') { - device(device_ip, name, cmd) + device(device_ip, name, cmd, controlPath) } } } @@ -161,10 +546,14 @@ node { env.PYTHONWARNINGS = "error" env.TEST_DIR = "/data/openpilot" env.SOURCE_DIR = "/data/openpilot_source/" + env.DEVICE_BUILD_DIR = "/device-build-cache/${(env.BRANCH_NAME ?: 'detached').replaceAll('[^A-Za-z0-9_.-]', '_')}" + env.DEVICE_BUILD_READY = "0" setupCredentials() - env.GIT_BRANCH = checkout(scm).GIT_BRANCH - env.GIT_COMMIT = checkout(scm).GIT_COMMIT + cleanOldWorkspaceBuildCache() + def scmVars = checkout(scm) + env.GIT_BRANCH = scmVars.GIT_BRANCH + env.GIT_COMMIT = scmVars.GIT_COMMIT def excludeBranches = ['__nightly', 'devel', 'devel-staging', 'release3', 'release3-staging', 'release-tici', 'release-tizi', 'release-tizi-staging', 'release-mici-staging', 'testing-closet*', 'hotfix-*'] @@ -199,7 +588,8 @@ node { } if (!env.BRANCH_NAME.matches(excludeRegex)) { - parallel ( + prepareBuiltTree() + parallel ( 'onroad tests': { deviceStage("onroad", "tizi-needs-can", ["UNSAFE=1"], [ step("build openpilot", "cd system/manager && ./build.py"), diff --git a/selfdrive/test/setup_device_ci.sh b/selfdrive/test/setup_device_ci.sh index 2a1442a20cce13..f0898d2e7091ec 100755 --- a/selfdrive/test/setup_device_ci.sh +++ b/selfdrive/test/setup_device_ci.sh @@ -62,27 +62,92 @@ sleep infinity EOF chmod +x $CONTINUE_PATH -safe_checkout() { - # completely clean TEST_DIR +if [ ! -z "$SKIP_GIT_CHECKOUT" ]; then + echo "device setup done, skipping git checkout" + exit 0 +fi - cd $SOURCE_DIR +fetch_commit() { + if git cat-file -e "$GIT_COMMIT^{commit}" 2>/dev/null; then + echo "$GIT_COMMIT already present" + return + fi + + git fetch --no-tags --no-recurse-submodules -j8 --verbose --depth 1 origin "$GIT_COMMIT" +} + +submodule_paths() { + git config --file .gitmodules --get-regexp path 2>/dev/null | awk '{print $2}' +} + +submodules_need_update() { + if [ -z "$OLD_HEAD" ]; then + return 0 + fi + + local paths + paths="$(submodule_paths)" + + for path in $paths; do + if [ ! -e "$path/.git" ]; then + return 0 + fi + done + + ! git diff --quiet "$OLD_HEAD" "$GIT_COMMIT" -- .gitmodules $paths +} + +lfs_needs_pull() { + if git lfs ls-files | awk '$2 == "-" { found=1 } END { exit found ? 0 : 1 }'; then + return 0 + fi + + if [ -z "$OLD_HEAD" ]; then + return 0 + fi + + if ! git diff --quiet "$OLD_HEAD" "$GIT_COMMIT" -- .gitattributes .lfsconfig; then + return 0 + fi + + git diff --name-only "$OLD_HEAD" "$GIT_COMMIT" -- | git check-attr --stdin filter | grep -q ': filter: lfs' +} + +checkout_common() { + local clean_args="$1" + local submodule_clean_args="$2" + + OLD_HEAD="$(git rev-parse HEAD 2>/dev/null || true)" # cleanup orphaned locks find .git -type f -name "*.lock" -exec rm {} + - git reset --hard - git fetch --no-tags --no-recurse-submodules -j4 --verbose --depth 1 origin $GIT_COMMIT - find . -maxdepth 1 -not -path './.git' -not -name '.' -not -name '..' -exec rm -rf '{}' \; - git reset --hard $GIT_COMMIT - git checkout $GIT_COMMIT - git clean -xdff - git submodule sync - git submodule foreach --recursive "git reset --hard && git clean -xdff" - git submodule update --init --recursive - git submodule foreach --recursive "git reset --hard && git clean -xdff" + git clean $clean_args + fetch_commit + GIT_LFS_SKIP_SMUDGE=1 git -c advice.detachedHead=false checkout --force --detach --no-recurse-submodules "$GIT_COMMIT" + git clean $clean_args + + if submodules_need_update; then + git submodule sync --recursive + git submodule update --init --recursive --force --jobs 8 + else + echo "submodules unchanged, skipping submodule update" + fi + git submodule foreach --recursive "git reset --hard && git clean $submodule_clean_args" - git lfs pull - (ulimit -n 65535 && git lfs prune) + if lfs_needs_pull; then + git lfs pull + else + echo "LFS files unchanged, skipping git lfs pull" + fi +} + +safe_checkout() { + # completely clean TEST_DIR + + cd $SOURCE_DIR + + checkout_common "-xdff" "-xdff" echo "git checkout done, t=$SECONDS" du -hs $SOURCE_DIR $SOURCE_DIR/.git @@ -95,20 +160,7 @@ unsafe_checkout() {( set -e cd $TEST_DIR - # cleanup orphaned locks - find .git -type f -name "*.lock" -exec rm {} + - - git fetch --no-tags --no-recurse-submodules -j8 --verbose --depth 1 origin $GIT_COMMIT - git checkout --force --no-recurse-submodules $GIT_COMMIT - git reset --hard $GIT_COMMIT - git clean -dff - git submodule sync - git submodule foreach --recursive "git reset --hard && git clean -df" - git submodule update --init --recursive - git submodule foreach --recursive "git reset --hard && git clean -df" - - git lfs pull - (ulimit -n 65535 && git lfs prune) + checkout_common "-dff" "-df" )} export GIT_PACK_THREADS=8