From 860bebd3547343fe4689dc19764824f4460fc7ee Mon Sep 17 00:00:00 2001 From: val Date: Tue, 19 May 2026 12:36:48 -0700 Subject: [PATCH 1/9] Add node_2026: Docker-based RPi4 streaming node with Pisound HAT Fully containerized implementation of the hydrophone streaming node. JACK, ffmpeg, and Python uploaders all run inside the container; the host only needs Docker. Includes internet outage recovery via catchup_s3.py and automatic restart via Docker restart: always. Co-Authored-By: Claude Sonnet 4.6 --- node_2026/.dockerignore | 7 ++ node_2026/.env.template | 20 ++++ node_2026/.gitignore | 6 + node_2026/Dockerfile | 46 ++++++++ node_2026/README.txt | 222 +++++++++++++++++++++++++++++++++++ node_2026/catchup_s3.py | 187 +++++++++++++++++++++++++++++ node_2026/docker-compose.yml | 35 ++++++ node_2026/setup.sh | 85 ++++++++++++++ node_2026/stream_sync.sh | 159 +++++++++++++++++++++++++ node_2026/upload_s3.py | 145 +++++++++++++++++++++++ 10 files changed, 912 insertions(+) create mode 100644 node_2026/.dockerignore create mode 100644 node_2026/.env.template create mode 100644 node_2026/.gitignore create mode 100644 node_2026/Dockerfile create mode 100644 node_2026/README.txt create mode 100644 node_2026/catchup_s3.py create mode 100644 node_2026/docker-compose.yml create mode 100644 node_2026/setup.sh create mode 100644 node_2026/stream_sync.sh create mode 100644 node_2026/upload_s3.py diff --git a/node_2026/.dockerignore b/node_2026/.dockerignore new file mode 100644 index 0000000..9569f67 --- /dev/null +++ b/node_2026/.dockerignore @@ -0,0 +1,7 @@ +.env +.env.template +.env_actual +.git +*.log +.venv/ +__pycache__/ diff --git a/node_2026/.env.template b/node_2026/.env.template new file mode 100644 index 0000000..cbc32d5 --- /dev/null +++ b/node_2026/.env.template @@ -0,0 +1,20 @@ +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_METADATA_SERVICE_TIMEOUT=5 +AWS_METADATA_SERVICE_NUM_ATTEMPTS=0 +REGION=us-west-2 +#BUCKET_TYPE=dev +BUCKET_TYPE=prod +#NODE_TYPE=research +NODE_TYPE=hls-only +NODE_NAME=rpi_orcasound_lab +NODE_LOOPBACK=false +SAMPLE_RATE=48000 +AUDIO_HW_ID=pisound +CHANNELS=2 +SYSLOG_URL=syslog://syslog-a.logdna.com:37043 +SYSLOG_STRUCTURED_DATA='logdna@48950 key="" tag="docker"' +FLAC_DURATION=30 +SEGMENT_DURATION=10 +LC_ALL=C.UTF-8 +NO_UPLOAD=true diff --git a/node_2026/.gitignore b/node_2026/.gitignore new file mode 100644 index 0000000..9d707e5 --- /dev/null +++ b/node_2026/.gitignore @@ -0,0 +1,6 @@ +.env +.env_actual +*.log +log*.txt +.venv/ +__pycache__/ diff --git a/node_2026/Dockerfile b/node_2026/Dockerfile new file mode 100644 index 0000000..3f31a53 --- /dev/null +++ b/node_2026/Dockerfile @@ -0,0 +1,46 @@ +# Node Dockerfile for hydrophone streaming +# Builds on standard Debian Trixie slim base (replaces unavailable orcastream/orcabase) + +FROM debian:trixie + +LABEL maintainer="Orcasound " + +ENV DEBIAN_FRONTEND=noninteractive + +###### Install system dependencies ##################################### + +RUN apt-get update && apt-get install -y --no-install-recommends \ + alsa-utils \ + jackd2 \ + jack-example-tools \ + libzita-alsa-pcmi0 \ + ffmpeg \ + python3-venv \ + python3-pip \ + python3 \ + curl \ + sudo \ + && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +###### Set up Python virtual environment ############################### + +RUN python3 -m venv /venv +RUN /venv/bin/pip install --upgrade pip +RUN /venv/bin/pip install boto3 inotify numpy + +# Make venv the default Python environment +ENV PATH="/venv/bin:$PATH" + +###### Audio group permissions ######################################### + +RUN usermod -aG audio root + +###### Set working directory and copy files ############################ + +WORKDIR /app +COPY . . + +RUN chmod +x stream_sync.sh +###### Runtime ######################################################### + +CMD ["./stream_sync.sh"] diff --git a/node_2026/README.txt b/node_2026/README.txt new file mode 100644 index 0000000..d6489fe --- /dev/null +++ b/node_2026/README.txt @@ -0,0 +1,222 @@ +================================================================================ +ORCASOUND HYDROPHONE NODE +================================================================================ +Hardware: Raspberry Pi 4 with Pisound HAT +OS: Raspberry Pi OS (Bookworm or Trixie) +Container: orcasound/orcanode_val_docker (built locally from Dockerfile) +================================================================================ + +OVERVIEW +-------- +This node captures audio from a Pisound HAT, segments it into HLS (.ts) files +using JACK + ffmpeg, and uploads them to S3. Everything except Docker runs +inside the container. Docker restarts the container automatically on crash or +reboot — no separate systemd service is needed. + +On startup stream_sync.sh: + 1. Waits for a sane system clock + 2. Discovers the Pisound ALSA device + 3. Starts jackd with the discovered hw address + 4. Launches ffmpeg to capture from JACK and write HLS segments + 5. Launches upload_s3.py to stream segments to S3 as they are written + 6. Launches catchup_s3.py (nice -n 10) to recover segments missed during + any internet outage + + +PREREQUISITES +------------- + - Raspberry Pi 4 with Pisound HAT installed and recognized by the OS + - Fresh Raspberry Pi OS image flashed to SD card + - SSH enabled, Pi connected to the internet + - AWS credentials with write access to the target S3 bucket + + +-------------------------------------------------------------------------------- +STEP 1 - FIRST BOOT CONFIGURATION +-------------------------------------------------------------------------------- + +Flash your SD card with Raspberry Pi Imager and set: + - Hostname (e.g. rpi-orcasound-lab) + - SSH enabled + - Username: pi (or your preferred username) + - Password + - WiFi SSID and password (if not using ethernet) + +Boot the Pi and SSH in: + ssh pi@ + + +-------------------------------------------------------------------------------- +STEP 2 - CLONE THE REPOSITORY +-------------------------------------------------------------------------------- + + sudo apt-get install -y git + git clone https://github.com/orcasound/orcanode.git ~/orcanode + cd ~/orcanode/node_val_docker + + +-------------------------------------------------------------------------------- +STEP 3 - CREATE THE .ENV FILE +-------------------------------------------------------------------------------- + +The .env file holds node-specific config and AWS credentials. It is never +baked into the Docker image — Docker Compose injects it at runtime. + + cp ~/orcanode/node_2026/.env.template ~/orcanode/node_2026/.env + nano ~/orcanode/node_2026/.env + +Required variables: + + NODE_NAME=rpi_orcasound_lab # unique name for this node + NODE_TYPE=hls-only # hls-only or research + AUDIO_HW_ID=pisound # sound card name — verify with: aplay -l + SAMPLE_RATE=48000 + CHANNELS=2 + SEGMENT_DURATION=10 # HLS segment length in seconds + FLAC_DURATION=30 # FLAC archive chunk length (research mode) + NODE_LOOPBACK=false # true to monitor audio on local output + BUCKET_TYPE=prod # prod, dev, or custom + AWS_ACCESS_KEY_ID= + AWS_SECRET_ACCESS_KEY= + AWS_METADATA_SERVICE_TIMEOUT=5 + AWS_METADATA_SERVICE_NUM_ATTEMPTS=0 + REGION=us-west-2 + SYSLOG_URL=syslog://syslog-a.logdna.com:37043 + SYSLOG_STRUCTURED_DATA='logdna@48950 key="" tag="docker"' + LC_ALL=C.UTF-8 + NO_UPLOAD=false # set to true to test pipeline without S3 + +NOTE: Set NO_UPLOAD=true during initial testing. Segments will accumulate +locally in /tmp//hls/ so you can verify the pipeline end-to-end +before enabling live uploads. + + +-------------------------------------------------------------------------------- +STEP 4 - RUN SETUP SCRIPT +-------------------------------------------------------------------------------- + +setup.sh installs Docker, fixes the Docker Hub IPv6 issue (common on Pi OS +Trixie), and adds the user to the docker and audio groups: + + cd ~/orcanode/node_val_docker + bash setup.sh + +The script reboots the Pi when complete. Wait for reboot, then SSH back in. + +Note: jackd, ffmpeg, and Python run inside the Docker container — setup.sh +does not install them on the host. + + +-------------------------------------------------------------------------------- +STEP 5 - BUILD AND START THE CONTAINER +-------------------------------------------------------------------------------- + +Build the image and start (first time only, or after code changes): + + cd ~/orcanode/node_val_docker + docker compose up -d --build + +After the first build, Docker manages the container automatically: + - restart: always — restarts on crash without any action needed + - Docker enabled at boot — container starts on every reboot + +Watch the startup logs: + + docker compose logs -f + +Healthy startup looks like: + Time looks sane: + Success! pisound found at index N. Using address: hw:N,0 + JACK is ready. + (then silence — ffmpeg and the uploaders run quietly) + + +-------------------------------------------------------------------------------- +STEP 6 - VERIFY THE PIPELINE +-------------------------------------------------------------------------------- + +Check that HLS segments are being generated locally: + + docker compose exec streaming ls -lh /tmp//hls/ + +Each .ts segment should be 150-300 KB. Watch them appear in real time: + + docker compose exec streaming watch -n 1 'ls -lh /tmp//hls/*/' + +If NO_UPLOAD=false, verify segments are reaching S3: + + aws s3 ls s3://audio-orcasound-net//hls/ --human-readable + +Check the upload log for RMS values (healthy signal = RMS > 100): + + docker compose logs -f + + +-------------------------------------------------------------------------------- +CONTAINER MANAGEMENT +-------------------------------------------------------------------------------- + + docker compose up -d # start (after first build) + docker compose down # stop + docker compose restart # restart + docker compose logs -f # follow live logs + docker compose up -d --build # rebuild image and restart (after code changes) + +Open a shell inside the running container: + docker compose exec streaming /bin/bash + +Check JACK port connections from inside the container: + docker compose exec streaming jack_lsp -c + +You should see system:capture_1/2 connected to ffjack:input_1/2. + + +-------------------------------------------------------------------------------- +INTERNET OUTAGE RECOVERY +-------------------------------------------------------------------------------- + +upload_s3.py leaves segments on disk when uploads fail. When connectivity +returns, catchup_s3.py finds stranded segments, generates a VOD manifest +(catchup.m3u8), and uploads everything at low priority (2s between segments). + +Disk guard: if stranded segments exceed 500 MB, the oldest are deleted first +to protect the SD card. + +No configuration required — this runs automatically alongside upload_s3.py. + + +-------------------------------------------------------------------------------- +TROUBLESHOOTING +-------------------------------------------------------------------------------- + +Docker pull/push fails ("network is unreachable"): + IPv6 issue on Pi OS Trixie. setup.sh fixes this automatically. If it + recurs, re-run setup.sh or manually add the Docker Hub IPv4 to /etc/hosts: + curl -4 -v https://registry-1.docker.io/v2/ 2>&1 | grep "Connected to" + echo " registry-1.docker.io" | sudo tee -a /etc/hosts + +JACK "Bus error" or "Cannot lock down memory": + The container needs a larger /dev/shm. Verify docker-compose.yml contains: + shm_size: '256m' + Rebuild and restart if you change it. + +Audio device not found (pisound): + Check the device is visible on the host: + aplay -l + Verify AUDIO_HW_ID in .env matches the card name shown by aplay -l. + +Segments are silent (RMS near 0): + JACK ports are not connected. Reconnect manually and restart: + docker compose exec streaming jack_connect system:capture_1 ffjack:input_1 + docker compose exec streaming jack_connect system:capture_2 ffjack:input_2 + docker compose restart + +.env not loading: + Verify no Windows line endings: + file ~/orcanode/node_val_docker/.env + If it shows "CRLF", convert it: + sed -i 's/\r//' ~/orcanode/node_val_docker/.env + +================================================================================ +For help, open an issue at: https://github.com/orcasound/orcanode +================================================================================ diff --git a/node_2026/catchup_s3.py b/node_2026/catchup_s3.py new file mode 100644 index 0000000..6590898 --- /dev/null +++ b/node_2026/catchup_s3.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +""" +Catches up HLS segment uploads missed during internet outages. +Runs alongside upload_s3.py at low priority (launched with nice -n 10). + +Segments that upload_s3.py failed to upload stay on disk. +Every SCAN_INTERVAL seconds this script finds segments older than +STALE_AGE, waits for connectivity, generates a VOD manifest, then +uploads everything at a throttled rate before deleting local copies. + +Disk guard: if stranded segments exceed MAX_STRANDED_BYTES, oldest +segments are deleted to protect the SD card. +""" + +import os +import sys +import time +import glob +import logging +import urllib.request +import boto3 + +NODE = os.environ["NODE_NAME"] +SEGMENT_DURATION = int(os.environ.get("SEGMENT_DURATION", "10").strip()) +BASEPATH = os.path.join("/tmp", NODE) +HLS_PATH = os.path.join(BASEPATH, "hls") + +BUCKET = "" +if "BUCKET_TYPE" in os.environ: + if os.environ["BUCKET_TYPE"] == "prod": + BUCKET = "audio-orcasound-net" + elif os.environ["BUCKET_TYPE"] == "custom": + BUCKET = os.environ["BUCKET_STREAMING"] + else: + BUCKET = "dev-streaming-orcasound-net" + +# A segment is stranded if it is older than this — gives upload_s3.py +# enough time to handle it first before catchup touches it. +STALE_AGE = SEGMENT_DURATION * 3 + +SCAN_INTERVAL = 30 # seconds between directory scans +CATCHUP_SLEEP = 2.0 # seconds between individual catch-up uploads +MAX_STRANDED_BYTES = 500 * 1024 * 1024 # 500 MB disk guard + +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler(sys.stdout) +handler.setFormatter(logging.Formatter("catchup.%(funcName)s: %(message)s")) +log.addHandler(handler) + + +def is_connected(): + try: + urllib.request.urlopen("https://s3.amazonaws.com", timeout=5) + return True + except Exception: + return False + + +def find_stranded_segments(): + """Return sorted list of .ts paths older than STALE_AGE.""" + now = time.time() + found = [] + for path in glob.glob(os.path.join(HLS_PATH, "*", "*.ts")): + try: + if now - os.path.getmtime(path) > STALE_AGE: + found.append(path) + except FileNotFoundError: + pass + return sorted(found) + + +def total_size(paths): + total = 0 + for p in paths: + try: + total += os.path.getsize(p) + except FileNotFoundError: + pass + return total + + +def enforce_disk_guard(stranded): + """Delete oldest stranded segments if total exceeds MAX_STRANDED_BYTES.""" + # Sort oldest first (by mtime) + aged = sorted(stranded, key=lambda p: os.path.getmtime(p)) + while total_size(aged) > MAX_STRANDED_BYTES and aged: + victim = aged.pop(0) + try: + os.remove(victim) + log.warning(f"Disk guard: deleted {os.path.basename(victim)}") + except FileNotFoundError: + pass + return aged # updated list after deletions + + +def write_catchup_manifest(ts_files, manifest_path): + """Write a VOD HLS manifest for the given segment files.""" + with open(manifest_path, "w") as f: + f.write("#EXTM3U\n") + f.write("#EXT-X-VERSION:3\n") + f.write(f"#EXT-X-TARGETDURATION:{SEGMENT_DURATION}\n") + for ts_file in ts_files: + f.write(f"#EXTINF:{SEGMENT_DURATION}.0,\n") + f.write(os.path.basename(ts_file) + "\n") + f.write("#EXT-X-ENDLIST\n") + + +def s3_upload(local_path, s3_key): + """Upload one file; return True on success.""" + try: + boto3.resource("s3").meta.client.upload_file(local_path, BUCKET, s3_key) + return True + except Exception as e: + log.warning(f"Upload failed {os.path.basename(local_path)}: {e}") + return False + + +def s3_key_for(local_path): + return os.path.relpath(local_path, "/tmp") + + +def catchup_session(ts_dir, ts_files): + """Upload the catchup manifest + segments for one timestamp directory.""" + timestamp = os.path.basename(ts_dir) + + manifest_path = os.path.join(ts_dir, "catchup.m3u8") + write_catchup_manifest(ts_files, manifest_path) + + manifest_key = s3_key_for(manifest_path) + if not s3_upload(manifest_path, manifest_key): + os.remove(manifest_path) + return False + os.remove(manifest_path) + log.info(f"Catchup manifest uploaded for {timestamp} ({len(ts_files)} segments)") + + for ts_file in ts_files: + if not is_connected(): + log.warning("Connectivity lost mid-catchup, pausing.") + return False + if s3_upload(ts_file, s3_key_for(ts_file)): + try: + os.remove(ts_file) + except FileNotFoundError: + pass + log.debug(f"Caught up: {os.path.basename(ts_file)}") + else: + return False + time.sleep(CATCHUP_SLEEP) + + return True + + +def _main(): + log.info("Catchup uploader started.") + while True: + time.sleep(SCAN_INTERVAL) + + stranded = find_stranded_segments() + if not stranded: + continue + + log.info(f"Found {len(stranded)} stranded segment(s), " + f"{total_size(stranded) // 1024} KB total.") + + stranded = enforce_disk_guard(stranded) + if not stranded: + continue + + if not is_connected(): + log.info("No internet yet, will retry.") + continue + + # Group by timestamp directory + by_dir: dict = {} + for f in stranded: + by_dir.setdefault(os.path.dirname(f), []).append(f) + + for ts_dir, ts_files in sorted(by_dir.items()): + if not is_connected(): + log.info("Lost connectivity, stopping catch-up round.") + break + catchup_session(ts_dir, ts_files) + + +if __name__ == "__main__": + _main() diff --git a/node_2026/docker-compose.yml b/node_2026/docker-compose.yml new file mode 100644 index 0000000..0a467f8 --- /dev/null +++ b/node_2026/docker-compose.yml @@ -0,0 +1,35 @@ +services: + streaming: + image: orcasound/orcanode_val_docker + build: . + # Overrides CMD in Dockerfile if needed, but keeping it explicit is good + command: ./stream_sync.sh + restart: always + env_file: .env + + # 1. CRITICAL: Bridge the host's Chrony to the container + volumes: + - /var/run/chrony:/var/run/chrony + - /etc/localtime:/etc/localtime:ro + - /usr/share/alsa:/usr/share/alsa:ro + + ports: + - "1234:1234" + - "8080:8080" + + devices: + - "/dev/snd:/dev/snd" + + # 2. Grant high-level hardware access + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + privileged: true + shm_size: '256m' + ulimits: + rtprio: 95 + memlock: + soft: -1 + hard: -1 diff --git a/node_2026/setup.sh b/node_2026/setup.sh new file mode 100644 index 0000000..97fecd6 --- /dev/null +++ b/node_2026/setup.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# One-time setup script for hydrophone streaming node (Docker version) +# Run once on a fresh Raspberry Pi OS (Bullseye, Bookworm, or Trixie) + +set -e # Exit immediately if any command fails + +# Detect the actual user (works whether run as root or with sudo) +REAL_USER=${SUDO_USER:-$USER} +REAL_HOME=$(eval echo "~$REAL_USER") +PROJECT_DIR="$REAL_HOME/orcanode/node_val_docker" + +echo "=== Hydrophone Node Setup (Docker) ===" +echo "Starting at $(date)" +echo "Setting up for user: $REAL_USER" + +# --- 1. SYSTEM UPDATE --- +echo "Updating system packages..." +sudo apt-get update +sudo apt-get upgrade -y + +# --- 2. DOCKER --- +echo "Installing Docker..." +curl -fsSL https://get.docker.com | sudo sh + +# Add user to docker group so docker runs without sudo +sudo usermod -aG docker "$REAL_USER" + +# Enable Docker to start on boot +sudo systemctl enable docker +sudo systemctl start docker + +# --- 3. FIX DOCKER IPV6 ISSUE --- +# Raspberry Pi OS Trixie resolves Docker Hub to IPv6 which is unreachable. +# This forces Docker to use IPv4 addresses instead. +echo "Fixing Docker IPv4 connectivity..." + +DOCKER_IP=$(curl -4 -s -v https://registry-1.docker.io/v2/ 2>&1 | grep "Connected to" | awk '{print $4}' | tr -d '()') +if [ -n "$DOCKER_IP" ]; then + grep -qF "registry-1.docker.io" /etc/hosts || echo "$DOCKER_IP registry-1.docker.io" | sudo tee -a /etc/hosts + echo "Added $DOCKER_IP for registry-1.docker.io to /etc/hosts" +else + echo "WARNING: Could not determine registry-1.docker.io IPv4 address." +fi + +DOCKER_AUTH_IP=$(curl -4 -s -v https://auth.docker.io/token 2>&1 | grep "Connected to" | awk '{print $4}' | tr -d '()') +if [ -n "$DOCKER_AUTH_IP" ]; then + grep -qF "auth.docker.io" /etc/hosts || echo "$DOCKER_AUTH_IP auth.docker.io" | sudo tee -a /etc/hosts + echo "Added $DOCKER_AUTH_IP for auth.docker.io to /etc/hosts" +else + echo "WARNING: Could not determine auth.docker.io IPv4 address." +fi + +sudo tee /etc/docker/daemon.json > /dev/null << 'DAEMONJSON' +{ + "ipv6": false, + "dns": ["8.8.8.8", "8.8.4.4"] +} +DAEMONJSON +sudo systemctl restart docker + +# --- 4. AUDIO GROUP PERMISSIONS --- +# Needed for /dev/snd device passthrough into the Docker container +sudo usermod -aG audio "$REAL_USER" + +echo "" +echo "=== Setup Complete ===" +echo "IMPORTANT: You must reboot before starting the container." +echo " Group membership changes (audio, docker) require a reboot to take effect." +echo "" +echo "After rebooting, build and start the container once:" +echo " cd $PROJECT_DIR" +echo " docker compose up -d --build" +echo "" +echo "After the first build, Docker manages the container automatically:" +echo " restart: always — restarts on crash" +echo " Docker enabled at boot — starts on every reboot" +echo "" +echo "Container management:" +echo " docker compose up -d # start" +echo " docker compose down # stop" +echo " docker compose logs -f # logs" +echo "" +echo "Rebooting in 10 seconds... (Ctrl-C to cancel)" +sleep 10 +sudo reboot diff --git a/node_2026/stream_sync.sh b/node_2026/stream_sync.sh new file mode 100644 index 0000000..d839dc7 --- /dev/null +++ b/node_2026/stream_sync.sh @@ -0,0 +1,159 @@ +#!/bin/bash +# Script for live DASH/HLS streaming lossy audio as AAC and/or archiving lossless audio as FLAC + +# Ensure system binaries are in PATH +export PATH=/usr/bin:/usr/local/bin:/usr/sbin:/bin:$PATH + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# --- 0. PREFLIGHT CHECKS --- +# The .env sourcing is commented out because Docker Compose handles variable injection. +# if [ -f "$SCRIPT_DIR/.env" ]; then +# set -a +# source "$SCRIPT_DIR/.env" +# set +a +# echo "Loaded .env from $SCRIPT_DIR" +# fi + +if ! command -v jackd &> /dev/null; then + echo "ERROR: jackd is not installed." + exit 1 +fi +if ! command -v jack_wait &> /dev/null; then + echo "ERROR: jack_wait is not installed." + exit 1 +fi +if ! command -v ffmpeg &> /dev/null; then + echo "ERROR: ffmpeg is not installed." + exit 1 +fi + +# --- 1. TIME SYNCHRONIZATION --- +wait_for_sync() { + echo "Checking for sane system time..." + local max_wait=60 + local elapsed=0 + + # Check if the year is 2025 or later + while [ $(date +%Y) -lt 2025 ]; do + if [ $elapsed -ge $max_wait ]; then + echo "ERROR: Time sync timed out, aborting." + exit 1 + fi + echo "Waiting for time sync (current year: $(date +%Y))..." + sleep 2 + elapsed=$((elapsed + 2)) + done + echo "Time looks sane: $(date)" +} + +wait_for_sync + +# --- 2. ACTIVATE VIRTUAL ENVIRONMENT --- +# Updated to use the absolute path within the container. +source /venv/bin/activate + +# --- 3. DYNAMIC AUDIO DEVICE DISCOVERY --- +echo "Searching for ALSA device: $AUDIO_HW_ID..." + +# Loop to find the hardware index (e.g., card 3) and set the correct hw address. +for i in $(seq 1 30); do + if aplay -l 2>/dev/null | grep -qi "$AUDIO_HW_ID"; then + # Extracts the number from a line like "card 3: pisound [pisound]". + CARD_NUM=$(aplay -l | grep -i "$AUDIO_HW_ID" | head -n 1 | awk -F'[: ]+' '{print $2}') + + if [ -n "$CARD_NUM" ]; then + SOUND_CARD="hw:$CARD_NUM,0" + echo "Success! $AUDIO_HW_ID found at index $CARD_NUM. Using address: $SOUND_CARD" + break + fi + fi + echo "Audio device $AUDIO_HW_ID not ready yet, attempt $i..." + sleep 1 +done + +if [ -z "$SOUND_CARD" ]; then + echo "ERROR: ALSA device $AUDIO_HW_ID not found after 30s, aborting." + exit 1 +fi + +# --- 4. VARIABLES & DIRECTORIES --- +timestamp=$(date +%s) +echo "Node started at $timestamp. Node name: $NODE_NAME. Sound card address: $SOUND_CARD" + +mkdir -p /tmp/$NODE_NAME/flac +mkdir -p /tmp/$NODE_NAME/hls/$timestamp +echo $timestamp > /tmp/$NODE_NAME/latest.txt + +STREAM_RATE=48000 +SAMPLE_RATE=${SAMPLE_RATE:-48000} + +# --- 5. SETUP JACK --- +# Start jackd using the discovered $SOUND_CARD index. +JACK_NO_AUDIO_RESERVATION=1 jackd -t 2000 -P 75 -m -s -d alsa -d $SOUND_CARD -r $SAMPLE_RATE -p 1024 -n 10 & + +echo "Waiting for JACK server to be ready..." +for i in $(seq 1 15); do + jack_wait -w -t 1 && break + if [ $i -eq 15 ]; then + echo "ERROR: JACK failed to start after 15s." + exit 1 + fi + sleep 1 +done +echo "JACK is ready." + +# --- 6. FFMPEG STREAMING --- +FFMPEG_PID="" + +if [ "$NODE_TYPE" = "research" ]; then + nice -n -10 ffmpeg -f jack -i ffjack \ + -f segment -segment_time "00:00:$FLAC_DURATION.00" -strftime 1 "/tmp/$NODE_NAME/flac/%Y-%m-%d_%H-%M-%S_$NODE_NAME-$SAMPLE_RATE-$CHANNELS.flac" \ + -f segment -segment_list "/tmp/$NODE_NAME/hls/$timestamp/live.m3u8" -segment_list_flags +live -segment_list_size 5 -segment_time $SEGMENT_DURATION -segment_format \ + mpegts -ar $STREAM_RATE -ac 2 -acodec aac "/tmp/$NODE_NAME/hls/$timestamp/live%03d.ts" \ + >/tmp/$NODE_NAME/ffmpeg.log 2>&1 & + FFMPEG_PID=$! +elif [ "$NODE_TYPE" = "hls-only" ]; then + nice -n -10 ffmpeg -f jack -i ffjack -f segment -segment_list "/tmp/$NODE_NAME/hls/$timestamp/live.m3u8" -segment_list_flags +live -segment_list_size 5 -segment_time $SEGMENT_DURATION -segment_format mpegts -ar $STREAM_RATE -ac $CHANNELS -threads 3 -acodec aac "/tmp/$NODE_NAME/hls/$timestamp/live%03d.ts" \ + >/tmp/$NODE_NAME/ffmpeg.log 2>&1 & + FFMPEG_PID=$! +else + echo "Unsupported NODE_TYPE. Please use research or hls-only." + exit 1 +fi + +# --- 7. CONNECT JACK PORTS --- +echo "Waiting for ffjack ports..." +for i in $(seq 1 15); do + jack_lsp | grep -q "ffjack:input_1" && break + sleep 1 +done + +if ! jack_lsp | grep -q "ffjack:input_1"; then + echo "ERROR: ffjack ports never appeared." + cat /tmp/$NODE_NAME/ffmpeg.log + exit 1 +fi + +jack_connect -s default system:capture_1 ffjack:input_1 +jack_connect -s default system:capture_2 ffjack:input_2 + +if [ "$NODE_LOOPBACK" = "true" ]; then + jack_connect system:capture_1 system:playback_1 + jack_connect system:capture_2 system:playback_2 +fi + +# Launch Python uploaders +if [ "${NO_UPLOAD:-false}" = "true" ]; then + echo "NO_UPLOAD=true, skipping S3 upload. Segments will accumulate in /tmp/$NODE_NAME/hls/" + wait $FFMPEG_PID +elif [ "$NODE_TYPE" = "research" ]; then + nice -n 10 python3 catchup_s3.py & + python3 upload_s3.py & + python3 upload_flac_s3.py +else + nice -n 10 python3 catchup_s3.py & + python3 upload_s3.py +fi + +echo "All processes started successfully." \ No newline at end of file diff --git a/node_2026/upload_s3.py b/node_2026/upload_s3.py new file mode 100644 index 0000000..965b8e2 --- /dev/null +++ b/node_2026/upload_s3.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# Based on https://github.com/gergnz/s3autoloader/blob/master/s3autoloader.py + +from boto3.s3.transfer import S3Transfer +import inotify.adapters +import logging +import logging.handlers +import boto3 +import numpy as np +import subprocess +import os +import sys + +NODE = os.environ["NODE_NAME"] +BASEPATH = os.path.join("/tmp", NODE) +PATH = os.path.join(BASEPATH, "hls") + +# REGION = os.environ["REGION"] +LOGLEVEL = logging.DEBUG + +log = logging.getLogger(__name__) +log.setLevel(LOGLEVEL) +handler = logging.StreamHandler(sys.stdout) +formatter = logging.Formatter('%(module)s.%(funcName)s: %(message)s') +handler.setFormatter(formatter) +log.addHandler(handler) + +BUCKET = "" +if "BUCKET_TYPE" in os.environ: + if(os.environ["BUCKET_TYPE"] == "prod"): + print("using production bucket") + BUCKET = 'audio-orcasound-net' + elif (os.environ["BUCKET_TYPE"] == "custom"): + print("using custom bucket") + BUCKET = os.environ["BUCKET_STREAMING"] + else: + BUCKET = "dev-streaming-orcasound-net" + + log.debug("hls bucket set to "+BUCKET) + + +def compute_rms(filepath): + """Decode audio from a .ts file using ffmpeg and compute RMS level.""" + try: + result = subprocess.run( + [ + 'ffmpeg', '-i', filepath, + '-ac', '1', # mix to mono for single RMS value + '-ar', '48000', # consistent sample rate + '-f', 's16le', # raw signed 16-bit little-endian PCM + '-' # output to stdout + ], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, # suppress ffmpeg's own output + timeout=10 + ) + if len(result.stdout) == 0: + log.warning(f'ffmpeg produced no audio output for {filepath}') + return None + samples = np.frombuffer(result.stdout, dtype=np.int16).astype(np.float32) + rms = np.sqrt(np.mean(samples ** 2)) + return rms + except subprocess.TimeoutExpired: + log.warning(f'ffmpeg timed out decoding {filepath}') + return None + except Exception as e: + log.warning(f'RMS computation failed for {filepath}: {e}') + return None + + +def s3_copy_file(path, filename): + uploadfile = os.path.join(path, filename) + + # Check file size first + try: + filesize = os.path.getsize(uploadfile) + except FileNotFoundError: + log.warning(f'SKIPPING {filename}: file already rotated by ffmpeg') + return + if filesize == 0: + log.warning(f'SKIPPING empty file: {filename}') + return + + # Compute and log RMS for .ts segment files + if filename.endswith('.ts'): + rms = compute_rms(uploadfile) + if rms is not None: + log.debug(f'file {filename} size: {filesize} bytes RMS: {rms:.1f}') + if rms < 1.0: + log.warning(f'Very low RMS ({rms:.2f}) in {filename} - possible silence or bad capture') + else: + log.debug(f'file {filename} size: {filesize} bytes RMS: could not compute') + else: + log.debug(f'file {filename} size: {filesize} bytes') + + log.debug('uploading file '+filename+' from '+path+' to bucket '+BUCKET) + log.debug(f'AWS_ACCESS_KEY_ID present: {"AWS_ACCESS_KEY_ID" in os.environ}') + log.debug(f'AWS_SECRET_ACCESS_KEY present: {"AWS_SECRET_ACCESS_KEY" in os.environ}') + try: + resource = boto3.resource('s3') + uploadpath = os.path.relpath(path, "/tmp") + uploadkey = os.path.join(uploadpath, filename) + log.debug('upload key: ' + uploadkey) + try: + resource.meta.client.upload_file(uploadfile, BUCKET, uploadkey) + except Exception as e: + log.critical('error uploading to S3: ' + str(e)) + return + + try: + os.remove(os.path.join(path, filename)) + except FileNotFoundError: + pass # ffmpeg may have already replaced or rotated the file + except Exception as e: + log.warning('error removing local file: ' + str(e)) + except: + e = sys.exc_info()[0] + log.critical('error uploading to S3: '+str(e)) + + +def _main(): + # latest.txt is uploaded after the first manifest, not at startup. + # This ensures the player can find segments when it reads the timestamp. + latest_txt_uploaded = False + i = inotify.adapters.InotifyTree(PATH) + try: + for event in i.event_gen(yield_nones=False): + (header, type_names, path, filename) = event + if type_names[0] == 'IN_CLOSE_WRITE': + if 'tmp' not in filename: + log.debug('Recieved a new file ' + filename) + s3_copy_file(path, filename) + if type_names[0] == 'IN_MOVED_TO': + log.debug('Recieved a new file ' + filename) + s3_copy_file(path, filename) + if not latest_txt_uploaded and filename.endswith('.ts'): + log.debug('First segment uploaded — now publishing latest.txt') + s3_copy_file(BASEPATH, 'latest.txt') + latest_txt_uploaded = True + finally: + log.debug('all done') + + +if __name__ == '__main__': + _main() From cf329453c1698533afdf424ce6b5731fb5ea10cb Mon Sep 17 00:00:00 2001 From: val Date: Sun, 24 May 2026 09:28:23 -0700 Subject: [PATCH 2/9] Update node_2026: native HLS muxer, rolling manifest, program_date_time --- node_2026/.env_template | 21 +++++++++++++++++++++ node_2026/.gitignore | 15 ++++++++++++--- node_2026/README.txt | 20 +++++++++++++++++++- node_2026/stream_sync.sh | 28 ++++++++++++++++++++++------ 4 files changed, 74 insertions(+), 10 deletions(-) create mode 100644 node_2026/.env_template diff --git a/node_2026/.env_template b/node_2026/.env_template new file mode 100644 index 0000000..4562fc2 --- /dev/null +++ b/node_2026/.env_template @@ -0,0 +1,21 @@ +AWSSECRETACCESSKEY=xxxxx +AWS_ACCESS_KEY_ID=xxxxx +AWS_SECRET_ACCESS_KEY=xxxxx +AWS_METADATA_SERVICE_TIMEOUT=5 +AWS_METADATA_SERVICE_NUM_ATTEMPTS=0 +REGION=us-west-2 +#BUCKET_TYPE=dev +BUCKET_TYPE=prod +#NODE_TYPE=research +NODE_TYPE=hls-only +NODE_NAME=rpi_orcasound_lab +NODE_LOOPBACK=true +SAMPLE_RATE=48000 +AUDIO_HW_ID=pisound +CHANNELS=2 +SYSLOG_URL=syslog://syslog-a.logdna.com:37043 +SYSLOG_STRUCTURED_DATA='logdna@48950 key="313dbd82f35ccbe462e6e3483984f464" tag="docker"' +FLAC_DURATION=30 +SEGMENT_DURATION=10 +LC_ALL=C.UTF-8 +NO_UPLOAD=true diff --git a/node_2026/.gitignore b/node_2026/.gitignore index 9d707e5..2c32738 100644 --- a/node_2026/.gitignore +++ b/node_2026/.gitignore @@ -1,6 +1,15 @@ +# Credentials — never commit .env -.env_actual -*.log -log*.txt + +# Generated / local tooling .venv/ __pycache__/ +*.pyc +*.log + +# IDE and editor state +.idea/ +.claude/ + +# Stale working copies +*\ (copy).* diff --git a/node_2026/README.txt b/node_2026/README.txt index d6489fe..d1f2c38 100644 --- a/node_2026/README.txt +++ b/node_2026/README.txt @@ -17,7 +17,11 @@ On startup stream_sync.sh: 1. Waits for a sane system clock 2. Discovers the Pisound ALSA device 3. Starts jackd with the discovered hw address - 4. Launches ffmpeg to capture from JACK and write HLS segments + 4. Launches ffmpeg to capture from JACK and write: + hls-only: HLS segments (.ts) + rolling 5-entry live.m3u8 + research: same HLS output, plus lossless FLAC archive chunks + Both modes embed absolute UTC timestamps (EXT-X-PROGRAM-DATE-TIME) + in the manifest so players and researchers can locate segments by time. 5. Launches upload_s3.py to stream segments to S3 as they are written 6. Launches catchup_s3.py (nice -n 10) to recover segments missed during any internet outage @@ -143,10 +147,24 @@ Each .ts segment should be 150-300 KB. Watch them appear in real time: docker compose exec streaming watch -n 1 'ls -lh /tmp//hls/*/' +In research mode, also check FLAC files are being written: + + docker compose exec streaming ls -lh /tmp//flac/ + +Each .flac file covers FLAC_DURATION seconds of lossless audio. + If NO_UPLOAD=false, verify segments are reaching S3: aws s3 ls s3://audio-orcasound-net//hls/ --human-readable +Segments are stored under a timestamp subdirectory, e.g.: + s3://audio-orcasound-net//hls//live000.ts + +The live manifest (live.m3u8) is a rolling window of the 5 most recent +segments (~50 seconds). Inspect it to confirm program_date_time tags: + + aws s3 cp s3://audio-orcasound-net//hls//live.m3u8 - + Check the upload log for RMS values (healthy signal = RMS > 100): docker compose logs -f diff --git a/node_2026/stream_sync.sh b/node_2026/stream_sync.sh index d839dc7..9edfa10 100644 --- a/node_2026/stream_sync.sh +++ b/node_2026/stream_sync.sh @@ -108,14 +108,30 @@ FFMPEG_PID="" if [ "$NODE_TYPE" = "research" ]; then nice -n -10 ffmpeg -f jack -i ffjack \ - -f segment -segment_time "00:00:$FLAC_DURATION.00" -strftime 1 "/tmp/$NODE_NAME/flac/%Y-%m-%d_%H-%M-%S_$NODE_NAME-$SAMPLE_RATE-$CHANNELS.flac" \ - -f segment -segment_list "/tmp/$NODE_NAME/hls/$timestamp/live.m3u8" -segment_list_flags +live -segment_list_size 5 -segment_time $SEGMENT_DURATION -segment_format \ - mpegts -ar $STREAM_RATE -ac 2 -acodec aac "/tmp/$NODE_NAME/hls/$timestamp/live%03d.ts" \ - >/tmp/$NODE_NAME/ffmpeg.log 2>&1 & + -f segment \ + -segment_time "00:00:$FLAC_DURATION.00" \ + -strftime 1 "/tmp/$NODE_NAME/flac/%Y-%m-%d_%H-%M-%S_$NODE_NAME-$SAMPLE_RATE-$CHANNELS.flac" \ + -ar $STREAM_RATE -ac $CHANNELS -acodec aac \ + -f hls \ + -hls_time $SEGMENT_DURATION \ + -hls_list_size 5 \ + -hls_flags program_date_time \ + -hls_segment_filename "/tmp/$NODE_NAME/hls/$timestamp/live%03d.ts" \ + "/tmp/$NODE_NAME/hls/$timestamp/live.m3u8" \ + >/tmp/$NODE_NAME/ffmpeg.log 2>&1 & FFMPEG_PID=$! elif [ "$NODE_TYPE" = "hls-only" ]; then - nice -n -10 ffmpeg -f jack -i ffjack -f segment -segment_list "/tmp/$NODE_NAME/hls/$timestamp/live.m3u8" -segment_list_flags +live -segment_list_size 5 -segment_time $SEGMENT_DURATION -segment_format mpegts -ar $STREAM_RATE -ac $CHANNELS -threads 3 -acodec aac "/tmp/$NODE_NAME/hls/$timestamp/live%03d.ts" \ - >/tmp/$NODE_NAME/ffmpeg.log 2>&1 & + nice -n -10 ffmpeg -f jack -i ffjack \ + -ar $STREAM_RATE \ + -ac $CHANNELS \ + -acodec aac \ + -f hls \ + -hls_time $SEGMENT_DURATION \ + -hls_list_size 5 \ + -hls_flags program_date_time \ + -hls_segment_filename "/tmp/$NODE_NAME/hls/$timestamp/live%03d.ts" \ + "/tmp/$NODE_NAME/hls/$timestamp/live.m3u8" \ + >/tmp/$NODE_NAME/ffmpeg.log 2>&1 & FFMPEG_PID=$! else echo "Unsupported NODE_TYPE. Please use research or hls-only." From 54b6fc58166e80e66a0eff8ff33da94c715ec30f Mon Sep 17 00:00:00 2001 From: val veirs Date: Sun, 24 May 2026 10:41:36 -0700 Subject: [PATCH 3/9] Replace AWS key placeholders in .env_template Updated AWS credentials placeholders in .env_template --- node_2026/.env_template | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/node_2026/.env_template b/node_2026/.env_template index 4562fc2..ce872e6 100644 --- a/node_2026/.env_template +++ b/node_2026/.env_template @@ -1,6 +1,5 @@ -AWSSECRETACCESSKEY=xxxxx -AWS_ACCESS_KEY_ID=xxxxx -AWS_SECRET_ACCESS_KEY=xxxxx +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= AWS_METADATA_SERVICE_TIMEOUT=5 AWS_METADATA_SERVICE_NUM_ATTEMPTS=0 REGION=us-west-2 From 39a0c7289c675d3e9a2248bccad127789a2600b2 Mon Sep 17 00:00:00 2001 From: val Date: Wed, 27 May 2026 21:09:39 -0700 Subject: [PATCH 4/9] Update node_2026: accumulating daily manifest and midnight restart cron Switch hls_list_size from 5 to 0 so live.m3u8 grows to include every segment written since the last container start. Add a midnight crontab in setup.sh to restart the container daily, giving each calendar day its own S3 timestamp directory with a complete manifest. This makes all segments discoverable by the orcasite server for spectrogram generation. Co-Authored-By: Claude Sonnet 4.6 --- node_2026/README.txt | 10 +++++++--- node_2026/setup.sh | 7 +++++++ node_2026/stream_sync.sh | 4 ++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/node_2026/README.txt b/node_2026/README.txt index d1f2c38..71d7813 100644 --- a/node_2026/README.txt +++ b/node_2026/README.txt @@ -18,7 +18,7 @@ On startup stream_sync.sh: 2. Discovers the Pisound ALSA device 3. Starts jackd with the discovered hw address 4. Launches ffmpeg to capture from JACK and write: - hls-only: HLS segments (.ts) + rolling 5-entry live.m3u8 + hls-only: HLS segments (.ts) + growing live.m3u8 (all segments for the day) research: same HLS output, plus lossless FLAC archive chunks Both modes embed absolute UTC timestamps (EXT-X-PROGRAM-DATE-TIME) in the manifest so players and researchers can locate segments by time. @@ -123,6 +123,9 @@ Build the image and start (first time only, or after code changes): After the first build, Docker manages the container automatically: - restart: always — restarts on crash without any action needed - Docker enabled at boot — container starts on every reboot + - crontab (installed by setup.sh) restarts at midnight each night so + each calendar day gets its own S3 timestamp directory and a complete + live.m3u8 covering only that day Watch the startup logs: @@ -160,8 +163,9 @@ If NO_UPLOAD=false, verify segments are reaching S3: Segments are stored under a timestamp subdirectory, e.g.: s3://audio-orcasound-net//hls//live000.ts -The live manifest (live.m3u8) is a rolling window of the 5 most recent -segments (~50 seconds). Inspect it to confirm program_date_time tags: +The live manifest (live.m3u8) grows throughout the day, accumulating every +segment since the last midnight restart. Inspect it to confirm program_date_time +tags: aws s3 cp s3://audio-orcasound-net//hls//live.m3u8 - diff --git a/node_2026/setup.sh b/node_2026/setup.sh index 97fecd6..abbc9cb 100644 --- a/node_2026/setup.sh +++ b/node_2026/setup.sh @@ -62,6 +62,13 @@ sudo systemctl restart docker # Needed for /dev/snd device passthrough into the Docker container sudo usermod -aG audio "$REAL_USER" +# --- 5. MIDNIGHT RESTART CRON JOB --- +# Restarts the container at midnight each day so ffmpeg opens a fresh +# timestamp directory on S3. This keeps each day's live.m3u8 to one +# day's worth of segments rather than growing forever. +CRON_JOB="0 0 * * * cd $PROJECT_DIR && docker compose restart" +( crontab -u "$REAL_USER" -l 2>/dev/null | grep -v "docker compose restart"; echo "$CRON_JOB" ) | crontab -u "$REAL_USER" - + echo "" echo "=== Setup Complete ===" echo "IMPORTANT: You must reboot before starting the container." diff --git a/node_2026/stream_sync.sh b/node_2026/stream_sync.sh index 9edfa10..5107b8a 100644 --- a/node_2026/stream_sync.sh +++ b/node_2026/stream_sync.sh @@ -114,7 +114,7 @@ if [ "$NODE_TYPE" = "research" ]; then -ar $STREAM_RATE -ac $CHANNELS -acodec aac \ -f hls \ -hls_time $SEGMENT_DURATION \ - -hls_list_size 5 \ + -hls_list_size 0 \ -hls_flags program_date_time \ -hls_segment_filename "/tmp/$NODE_NAME/hls/$timestamp/live%03d.ts" \ "/tmp/$NODE_NAME/hls/$timestamp/live.m3u8" \ @@ -127,7 +127,7 @@ elif [ "$NODE_TYPE" = "hls-only" ]; then -acodec aac \ -f hls \ -hls_time $SEGMENT_DURATION \ - -hls_list_size 5 \ + -hls_list_size 0 \ -hls_flags program_date_time \ -hls_segment_filename "/tmp/$NODE_NAME/hls/$timestamp/live%03d.ts" \ "/tmp/$NODE_NAME/hls/$timestamp/live.m3u8" \ From b3cdcfdfe0e575a2bffe0a9cd99031aaca9f1c46 Mon Sep 17 00:00:00 2001 From: val Date: Thu, 28 May 2026 11:35:14 -0700 Subject: [PATCH 5/9] Add repair_manifests.py to fix incomplete historical manifests When hls_list_size was 5, live.m3u8 only retained the last 5 segments of each session, leaving earlier segments undiscoverable by the orcasite server. repair_manifests.py scans all S3 timestamp directories, compares .ts file count against manifest entries, and reconstructs a complete live.m3u8 with approximate EXT-X-PROGRAM-DATE-TIME tags derived by anchoring on the exact timestamps still present in the tail of the old manifest. README_REPAIR.md documents the algorithm and usage. Co-Authored-By: Claude Sonnet 4.6 --- node_2026/README_REPAIR.md | 160 ++++++++++++++++ node_2026/repair_manifests.py | 334 ++++++++++++++++++++++++++++++++++ 2 files changed, 494 insertions(+) create mode 100644 node_2026/README_REPAIR.md create mode 100644 node_2026/repair_manifests.py diff --git a/node_2026/README_REPAIR.md b/node_2026/README_REPAIR.md new file mode 100644 index 0000000..fc56a30 --- /dev/null +++ b/node_2026/README_REPAIR.md @@ -0,0 +1,160 @@ +# repair_manifests.py — HLS Manifest Repair Tool + +## Background + +Each recording session creates a timestamp directory on S3 such as: + +``` +s3://audio-orcasound-net/rpi_orcasound_lab/hls/1748383745/ +``` + +Inside that directory, `upload_s3.py` uploads every `.ts` audio segment and +a `live.m3u8` playlist. Until May 2026 the playlist was a **rolling window** +(ffmpeg flag `-hls_list_size 5`), meaning it only ever listed the 5 most +recent segments. As new segments arrived, old ones were silently dropped from +the manifest and the manifest was overwritten on S3. + +The result: every session directory contains dozens or hundreds of `.ts` files +but a `live.m3u8` that references only the **last few** of them. The +orcasite server reads `live.m3u8` to discover segments for spectrogram +generation, so those early segments were effectively invisible. + +`repair_manifests.py` fixes this by reconstructing a complete `live.m3u8` +for every affected session directory. + +--- + +## How the repair works + +For each timestamp directory the script: + +1. **Lists all `.ts` objects** on S3 and sorts them by sequence number + (e.g. `live000.ts`, `live001.ts`, …). + +2. **Fetches the existing `live.m3u8`** and parses it for: + - `EXT-X-PROGRAM-DATE-TIME` — exact UTC timestamps for the segments that + were still in the rolling window when the session ended. + - `EXTINF` durations — used to compute the average segment length + (typically ~10.005 s). + +3. **Finds the anchor**: the earliest segment in the existing manifest that + has an exact `EXT-X-PROGRAM-DATE-TIME` tag. If no tag exists, the Unix + timestamp embedded in the directory name is used as a fallback anchor at + sequence 0. + +4. **Derives approximate times** for every missing segment by stepping from + the anchor: + + ``` + time(N) ≈ anchor_time + (N − anchor_seq) × avg_duration + ``` + + Segments that were already in the manifest keep their **exact** timestamps. + Earlier segments get timestamps marked `.000Z` (whole-second precision) to + signal they are approximate. + +5. **Writes a new complete `live.m3u8`** covering every `.ts` file, + with `EXT-X-MEDIA-SEQUENCE:0` and, for completed sessions, + `#EXT-X-ENDLIST`. The live (most recent) directory never gets + `EXT-X-ENDLIST` since recording is still in progress. + +6. **Uploads the repaired manifest** back to S3, overwriting the old one. + Read operations use the public bucket endpoint (no credentials needed). + Write operations require AWS credentials (see below). + +--- + +## Prerequisites + +```bash +pip3 install boto3 +``` + +AWS credentials must be configured for write access to the bucket. +The standard locations work: `~/.aws/credentials`, environment variables +(`AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`), or an IAM instance role. + +--- + +## Usage + +``` +python3 repair_manifests.py [--node NODE] [--bucket BUCKET] + [--days N] [--stop-at TIMESTAMP] + [--dry-run] +``` + +### Options + +| Flag | Default | Description | +|---|---|---| +| `--node` | `rpi_orcasound_lab` | Node name (S3 path prefix) | +| `--bucket` | `audio-orcasound-net` | S3 bucket name | +| `--days N` | 0 (all) | Only examine directories from the last N days | +| `--stop-at TIMESTAMP` | — | Stop when reaching directories older than this Unix timestamp | +| `--dry-run` | off | Show what would change without uploading anything | + +### Typical workflow + +```bash +# 1. Preview — see what is broken without changing anything +python3 repair_manifests.py --dry-run + +# 2. Repair just the last two weeks first as a sanity check +python3 repair_manifests.py --days 14 --dry-run +python3 repair_manifests.py --days 14 + +# 3. Repair everything +python3 repair_manifests.py +``` + +### Sample output + +``` +Scanning s3://audio-orcasound-net/rpi_orcasound_lab/hls/ (dry_run=False) +Found 47 directories to examine + + [fixed] *live* 1779942622: fixed: 18 → 18 segments + [fixed] 1779856222: fixed: 5 → 8640 segments + [fixed] 1779769822: fixed: 5 → 8637 segments + [ok ] 1779683422: 8640 segments, manifest complete + [skip ] 1748383745: no .ts files + ... + +Done. fixed=44 ok=2 skip=1 error=0 +``` + +Status meanings: + +| Status | Meaning | +|---|---| +| `fixed` | Manifest was incomplete — now repaired and uploaded | +| `ok` | Manifest already covered all segments — untouched | +| `skip` | Directory has no `.ts` files or an unparseable name | +| `error` | Upload failed — check AWS credentials and bucket permissions | + +--- + +## Accuracy of approximate timestamps + +Timestamps for segments that fell outside the old rolling window are +approximate. The error accumulates at roughly the difference between the +actual segment duration and the average: + +- ffmpeg produces very consistent segment lengths (~10.005 s ± 0.001 s) +- For a session that lost 8 000 segments before the anchor, the accumulated + error at segment 0 is typically **under 10 seconds** + +This is sufficient for the orcasite server to locate segments by time and +generate spectrograms. Exact timestamps are preserved for the tail of each +session (whatever was still in the original rolling manifest). + +--- + +## Going forward + +As of the May 2026 update, `stream_sync.sh` uses `-hls_list_size 0` and the +container restarts at midnight via crontab. Each calendar day now gets its +own timestamp directory with a `live.m3u8` that grows to include every segment +for that day. `repair_manifests.py` is only needed for sessions recorded +before that change. diff --git a/node_2026/repair_manifests.py b/node_2026/repair_manifests.py new file mode 100644 index 0000000..14604ca --- /dev/null +++ b/node_2026/repair_manifests.py @@ -0,0 +1,334 @@ +#!/usr/bin/env python3 +""" +repair_manifests.py + +Scans S3 timestamp directories for a node and repairs any live.m3u8 that +covers fewer segments than the directory actually contains. This fixes +sessions recorded when hls_list_size was small (e.g. 5), leaving most +segments with no manifest entry. + +For each missing segment the script derives an approximate +EXT-X-PROGRAM-DATE-TIME by anchoring on the exact timestamp of the first +segment already in the manifest and stepping backwards by the average +EXTINF duration. Fallback: use the Unix timestamp in the directory name. + +The most recent (live) directory is never given EXT-X-ENDLIST. +All older directories receive it so players know the stream is complete. + +Usage: + python3 repair_manifests.py [--node NODE_NAME] [--bucket BUCKET] + [--days N] [--dry-run] + +Defaults: + --node rpi_orcasound_lab + --bucket audio-orcasound-net + --days all directories +""" + +import argparse +import re +import sys +from datetime import datetime, timezone, timedelta + +import boto3 +from botocore import UNSIGNED +from botocore.config import Config +from botocore.exceptions import ClientError + +SEGMENT_DURATION_DEFAULT = 10.0 + + +# --------------------------------------------------------------------------- +# S3 helpers +# --------------------------------------------------------------------------- + +def list_timestamp_dirs(s3, bucket, node_name): + """Return sorted list of timestamp prefixes like 'node/hls/1748383745/'.""" + prefixes = [] + paginator = s3.get_paginator('list_objects_v2') + for page in paginator.paginate( + Bucket=bucket, + Prefix=f"{node_name}/hls/", + Delimiter='/' + ): + for cp in page.get('CommonPrefixes', []): + prefixes.append(cp['Prefix']) + return sorted(prefixes) + + +def list_ts_files(s3, bucket, prefix): + """Return .ts filenames in prefix, sorted by embedded sequence number.""" + files = [] + paginator = s3.get_paginator('list_objects_v2') + for page in paginator.paginate(Bucket=bucket, Prefix=prefix): + for obj in page.get('Contents', []): + name = obj['Key'].split('/')[-1] + if name.endswith('.ts'): + files.append(name) + + def seq(name): + m = re.search(r'(\d+)', name) + return int(m.group(1)) if m else 0 + + return sorted(files, key=seq) + + +def fetch_object(s3, bucket, key): + """Return object body as str, or None if not found.""" + try: + resp = s3.get_object(Bucket=bucket, Key=key) + return resp['Body'].read().decode('utf-8') + except ClientError as e: + if e.response['Error']['Code'] in ('NoSuchKey', '404'): + return None + raise + + +def put_object(s3, bucket, key, body): + s3.put_object( + Bucket=bucket, + Key=key, + Body=body.encode('utf-8'), + ContentType='application/x-mpegurl', + ) + + +# --------------------------------------------------------------------------- +# Manifest parsing +# --------------------------------------------------------------------------- + +def parse_manifest(content): + """ + Returns dict: + target_duration : int + media_sequence : int + segments : list of {filename, duration, pdt} + pdt is the EXT-X-PROGRAM-DATE-TIME string or None + """ + segments = [] + target_duration = 10 + media_sequence = 0 + pending_pdt = None + pending_duration = None + + for raw in content.splitlines(): + line = raw.strip() + if line.startswith('#EXT-X-TARGETDURATION:'): + try: + target_duration = int(line.split(':', 1)[1]) + except ValueError: + pass + elif line.startswith('#EXT-X-MEDIA-SEQUENCE:'): + try: + media_sequence = int(line.split(':', 1)[1]) + except ValueError: + pass + elif line.startswith('#EXT-X-PROGRAM-DATE-TIME:'): + pending_pdt = line.split(':', 1)[1] + elif line.startswith('#EXTINF:'): + try: + pending_duration = float(line[8:].split(',')[0]) + except ValueError: + pending_duration = SEGMENT_DURATION_DEFAULT + elif line.endswith('.ts') and not line.startswith('#'): + segments.append({ + 'filename': line, + 'duration': pending_duration or SEGMENT_DURATION_DEFAULT, + 'pdt': pending_pdt, + }) + pending_pdt = None + pending_duration = None + + return { + 'target_duration': target_duration, + 'media_sequence': media_sequence, + 'segments': segments, + } + + +# --------------------------------------------------------------------------- +# Manifest reconstruction +# --------------------------------------------------------------------------- + +def seq_num(filename): + m = re.search(r'(\d+)', filename) + return int(m.group(1)) if m else 0 + + +def build_manifest(all_ts, parsed, session_unix, add_endlist): + """ + Build a complete manifest string covering every file in all_ts. + + Exact PDT tags are kept for segments already in parsed['segments']. + Approximate PDTs are derived for the rest by anchoring on the earliest + known exact time and stepping by the average EXTINF duration. + """ + target_duration = parsed['target_duration'] + existing = {s['filename']: s for s in parsed['segments']} + + # Average duration from existing entries + durations = [s['duration'] for s in parsed['segments']] + avg_dur = sum(durations) / len(durations) if durations else SEGMENT_DURATION_DEFAULT + + # Anchor: earliest segment in the existing manifest that has a PDT tag + ref_dt = None + ref_sn = None + for seg in parsed['segments']: + if seg['pdt']: + try: + dt = datetime.fromisoformat(seg['pdt'].replace('Z', '+00:00')) + sn = seq_num(seg['filename']) + if ref_dt is None or sn < ref_sn: + ref_dt = dt + ref_sn = sn + except ValueError: + pass + + # Fallback anchor: directory Unix timestamp → seq 0 + if ref_dt is None: + ref_dt = datetime.fromtimestamp(session_unix, tz=timezone.utc) + ref_sn = 0 + + lines = [ + '#EXTM3U', + '#EXT-X-VERSION:3', + f'#EXT-X-TARGETDURATION:{target_duration}', + '#EXT-X-MEDIA-SEQUENCE:0', + ] + + for filename in all_ts: + sn = seq_num(filename) + if filename in existing and existing[filename]['pdt']: + pdt = existing[filename]['pdt'] + duration = existing[filename]['duration'] + else: + offset = (sn - ref_sn) * avg_dur + approx = ref_dt + timedelta(seconds=offset) + pdt = approx.strftime('%Y-%m-%dT%H:%M:%S.000Z') + duration = avg_dur + + lines.append(f'#EXT-X-PROGRAM-DATE-TIME:{pdt}') + lines.append(f'#EXTINF:{duration:.6f},') + lines.append(filename) + + if add_endlist: + lines.append('#EXT-X-ENDLIST') + + return '\n'.join(lines) + '\n' + + +# --------------------------------------------------------------------------- +# Per-directory processing +# --------------------------------------------------------------------------- + +def process_dir(s3, bucket, prefix, is_live, dry_run, s3_write=None): + """ + Returns (status, message) where status is one of: + 'fixed' — manifest was incomplete and has been repaired + 'ok' — manifest already covers all segments + 'skip' — directory skipped (no .ts files or bad name) + 'error' — something went wrong + """ + parts = prefix.rstrip('/').split('/') + try: + session_unix = int(parts[-1]) + except ValueError: + return 'skip', f'cannot parse timestamp from {prefix}' + + all_ts = list_ts_files(s3, bucket, prefix) + if not all_ts: + return 'skip', 'no .ts files' + + manifest_key = prefix + 'live.m3u8' + content = fetch_object(s3, bucket, manifest_key) + + if content: + parsed = parse_manifest(content) + else: + parsed = {'target_duration': 10, 'media_sequence': 0, 'segments': []} + + have = len(parsed['segments']) + need = len(all_ts) + + if have >= need: + return 'ok', f'{need} segments, manifest complete' + + add_endlist = not is_live + new_content = build_manifest(all_ts, parsed, session_unix, add_endlist) + + if not dry_run: + try: + put_object(s3_write or s3, bucket, manifest_key, new_content) + except Exception as e: + return 'error', str(e) + + verb = 'would fix' if dry_run else 'fixed' + return 'fixed', f'{verb}: {have} → {need} segments' + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--node', default='rpi_orcasound_lab', + help='Node name (S3 prefix, default: rpi_orcasound_lab)') + parser.add_argument('--bucket', default='audio-orcasound-net', + help='S3 bucket (default: audio-orcasound-net)') + parser.add_argument('--days', type=int, default=0, + help='Only process directories from the last N days (0 = all)') + parser.add_argument('--stop-at', type=int, default=0, metavar='TIMESTAMP', + help='Stop processing at this directory timestamp (inclusive); ' + 'directories older than this are skipped') + parser.add_argument('--dry-run', action='store_true', + help='Show what would change without uploading anything') + args = parser.parse_args() + + s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED)) + s3_write = None if args.dry_run else boto3.client('s3') + + print(f"Scanning s3://{args.bucket}/{args.node}/hls/ (dry_run={args.dry_run})") + dirs = list_timestamp_dirs(s3, args.bucket, args.node) + if not dirs: + print("No timestamp directories found.") + sys.exit(0) + + # Optionally limit to recent N days + if args.days > 0: + cutoff = datetime.now(tz=timezone.utc) - timedelta(days=args.days) + cutoff_unix = int(cutoff.timestamp()) + dirs = [d for d in dirs + if _dir_unix(d) >= cutoff_unix] + + print(f"Found {len(dirs)} director{'y' if len(dirs)==1 else 'ies'} to examine\n") + + most_recent = dirs[-1] if dirs else None + dirs = list(reversed(dirs)) + counts = {'fixed': 0, 'ok': 0, 'skip': 0, 'error': 0} + + for prefix in dirs: + ts_str = prefix.rstrip('/').split('/')[-1] + if args.stop_at and _dir_unix(prefix) < args.stop_at: + print(f" Reached stop-at threshold ({args.stop_at}), stopping.") + break + is_live = (prefix == most_recent) + status, msg = process_dir(s3, args.bucket, prefix, is_live, args.dry_run, s3_write) + tag = '*live*' if is_live else ' ' + print(f" [{status:5s}] {tag} {ts_str}: {msg}") + counts[status] += 1 + + print(f"\nDone. fixed={counts['fixed']} ok={counts['ok']} " + f"skip={counts['skip']} error={counts['error']}") + + +def _dir_unix(prefix): + try: + return int(prefix.rstrip('/').split('/')[-1]) + except ValueError: + return 0 + + +if __name__ == '__main__': + main() From 755ee2e3256a35978d5dd73c14c38d88b3bca2d7 Mon Sep 17 00:00:00 2001 From: val Date: Wed, 5 Aug 2026 09:45:07 -0700 Subject: [PATCH 6/9] node_2026: convert README to Markdown, add Tailscale setup and clone guide - Convert README.txt to README.md with proper Markdown formatting - Add Step 2: Install and Configure Tailscale, so field-deployed nodes get stable remote SSH access instead of relying on a LAN IP - Add CLONE_README.md documenting how to safely turn an SD-card clone into an independent node (unique NODE_NAME, machine-id/SSH host key regeneration, and resetting the cloned Tailscale identity so it doesn't collide with the source node on the tailnet) Co-Authored-By: Claude Sonnet 5 --- node_2026/CLONE_README.md | 271 +++++++++++++++++++++++++ node_2026/README.md | 402 ++++++++++++++++++++++++++++++++++++++ node_2026/README.txt | 244 ----------------------- 3 files changed, 673 insertions(+), 244 deletions(-) create mode 100644 node_2026/CLONE_README.md create mode 100644 node_2026/README.md delete mode 100644 node_2026/README.txt diff --git a/node_2026/CLONE_README.md b/node_2026/CLONE_README.md new file mode 100644 index 0000000..0f7311e --- /dev/null +++ b/node_2026/CLONE_README.md @@ -0,0 +1,271 @@ +# CLONE_README.md — Turning an SD-Card Clone into a New Node + +## Background + +The usual way to provision a new hydrophone node is to flash a fresh SD +card and walk through `README.txt` end to end (`setup.sh`, `.env`, Docker +build). That takes ~30-45 minutes per Pi because of package installs and +the Docker image build. + +A faster path: image a **known-good, already-built SD card** (Docker +installed, image already built, `orcanode` repo cloned) and clone that +image onto new SD cards for additional Pis. This skips the slow parts — +but a raw SD-card clone is a *bit-for-bit copy*, which means the new Pi +boots up claiming to be an exact duplicate of the original: same +hostname, same machine-id, same SSH host keys, same S3 `NODE_NAME`, and +— critically — the same **Tailscale identity**. + +If you skip the de-duplication steps below, two symptoms show up: + +- The new Pi's audio segments overwrite the original node's data on S3 + (they'd upload to the same `NODE_NAME` prefix). +- Tailscale treats the clone as the *same machine* re-appearing at a new + IP. Instead of a new device on your tailnet, you get one flapping + entry — whichever Pi checked in most recently "wins" and the other + drops offline. + +This doc covers de-duplicating a clone and bringing it up as a distinct, +independent node. + +--- + +## What's already on the clone + +Assuming the source SD card was a working `node_2026` install: + +- Raspberry Pi OS, Docker, and the `docker/audio` group memberships from + `setup.sh` +- `~/orcanode/node_2026/` checked out, with the Docker image + already built (`docker compose up -d --build` has been run at least + once) +- A working `.env` with the **original** node's `NODE_NAME` and AWS + credentials +- Tailscale installed and authenticated as the **original** node +- The container is likely still running from before the card was imaged + — it will start immediately using the old `.env` on first boot + +--- + +## Step 1 — Flash the clone and boot it standalone + +Flash the cloned image to the new SD card (Raspberry Pi Imager, `dd`, +balenaEtcher — whatever you used to make the clone in the first place). + +Before inserting it into the new Pi: + +- **Do not** boot the new Pi on the same network as the original while + both still share the same Tailscale identity — the identity collision + happens the moment `tailscaled` on the new Pi calls home. It's not + destructive, but it will bounce the original node offline until you + fix it, so do the fixes below before letting the new Pi sit online for + long. +- If you used Raspberry Pi Imager to write the clone, use its "Edit + Settings" (gear icon) to preset a new **hostname** and re-enable SSH + before first boot — this saves a step, but does not fix Tailscale or + the SSH host keys, which are handled below. + +Boot the new Pi and get a shell on it — either directly (keyboard/HDMI), +or via the LAN IP shown in your router's DHCP client list (do this over +plain SSH/local network, not Tailscale, since Tailscale isn't safe to +rely on yet): + +```bash +ssh pi@ +``` + +--- + +## Step 2 — Fix machine identity before anything else + +These make the new Pi behave as its own device instead of a duplicate of +the source. Do this before touching Tailscale. + +**Regenerate the machine-id** (used by systemd, DHCP client identifiers, +and some other services to distinguish hosts): + +```bash +sudo rm -f /etc/machine-id +sudo systemd-machine-id-setup +``` + +**Regenerate SSH host keys** (a cloned Pi ships with the *same* host +keys as the source — SSH clients that already trust the original will +throw host-key-mismatch warnings, or worse, silently trust the wrong +box): + +```bash +sudo rm -f /etc/ssh/ssh_host_* +sudo dpkg-reconfigure openssh-server +sudo systemctl restart ssh +``` + +**Set a unique hostname** (pick something that matches the new +`NODE_NAME` you'll set in Step 3, e.g. `rpi-orcasound-bush-point`): + +```bash +sudo raspi-config nonint do_hostname rpi-orcasound-bush-point +sudo reboot +``` + +SSH back in after the reboot (still over LAN, not Tailscale) using the +new hostname or the same LAN IP. + +--- + +## Step 3 — Edit `.env` for the new node + +Stop the container first so it isn't uploading under the old identity +while you edit: + +```bash +cd ~/orcanode/node_2026 +docker compose down +nano .env +``` + +| Variable | Action | Why | +|---|---|---| +| `NODE_NAME` | **Change — must be unique** | This is the S3 path prefix (`s3:////hls/...`). Two nodes sharing a name will overwrite each other's segments. | +| `AUDIO_HW_ID` | Verify | Should still be `pisound` if this Pi also has a Pisound HAT; confirm with `aplay -l` since HAT enumeration order can vary between boards. | +| `NODE_TYPE` | Verify | `hls-only` or `research` — set per what this node should do, independent of the source node's setting. | +| `NODE_LOOPBACK` | Verify | Local-monitoring preference for this physical install, not necessarily the same as the source. | +| `BUCKET_TYPE` | Usually unchanged | Keep `prod` unless this new node is a test/dev deployment. | +| `NO_UPLOAD` | Set `true` temporarily | Recommended for first boot — verify segments generate locally before enabling live S3 upload with a brand-new `NODE_NAME`. Flip to `false` once verified (see Step 6). | +| `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` | Usually unchanged | Same bucket, same credentials — unless this node should log in under a separate IAM identity. | +| `SYSLOG_STRUCTURED_DATA` | Usually unchanged | Shared LogDNA ingestion key; logs from all nodes land in the same place, distinguished by `NODE_NAME` in the log content. | +| Everything else (`SAMPLE_RATE`, `CHANNELS`, `SEGMENT_DURATION`, `FLAC_DURATION`, `REGION`, `LC_ALL`) | Usually unchanged | Hardware/format constants, not node-specific. | + +Do **not** rebuild the Docker image for a `.env` change — `.env` is +injected at container start via `env_file:` in `docker-compose.yml`, so +`docker compose up -d` alone picks up the new values. + +--- + +## Step 4 — Reset and re-initialize Tailscale + +This is the step that actually resolves the identity collision. Cloning +the SD card copied `/var/lib/tailscale/tailscaled.state`, which holds +the node's private key — that key *is* the machine's identity as far as +Tailscale is concerned. Wipe it and re-authenticate to mint a new +identity: + +```bash +sudo systemctl stop tailscaled +sudo rm -f /var/lib/tailscale/tailscaled.state +sudo systemctl start tailscaled +sudo tailscale up --hostname=rpi-orcasound-bush-point +``` + +- This prints an authentication URL — open it in a browser and approve + the new machine under your tailnet. +- `--hostname` sets the name Tailscale shows in the admin console; + match it to what you set in `raspi-config` in Step 2 so the LAN + hostname and Tailscale name agree. +- If you'd rather rename after the fact instead of via the flag, you can + approve first and then rename it from + `https://login.tailscale.com/admin/machines`. + +Confirm it registered as a **new, separate** device: + +```bash +tailscale status +``` + +You should see the new Pi listed by its new hostname with its own +`100.x.x.x` address, and — check the admin console — the **original** +node should still be online and unaffected (this confirms the identity +split actually worked; if the original dropped offline when the new one +came up, the state file wasn't fully cleared — repeat the `rm` / +`tailscale up` steps). + +Get the new Pi's Tailscale IP for later reference: + +```bash +tailscale ip -4 +``` + +--- + +## Step 5 — Bring the streaming container up + +```bash +cd ~/orcanode/node_2026 +docker compose up -d +``` + +(No `--build` needed — the image was already built on the source Pi and +cloned along with the rest of the SD card. Only rebuild if you've also +changed code, not just `.env`.) + +Watch the logs for a healthy startup: + +```bash +docker compose logs -f +``` + +Expect: + +``` +Time looks sane: +Success! pisound found at index N. Using address: hw:N,0 +JACK is ready. +``` + +--- + +## Step 6 — Verify the new node end-to-end + +**Tailscale admin console** (`https://login.tailscale.com/admin/machines`): +the new node appears as its own entry, distinct from the source, with +its own IP and last-seen time ticking forward. + +**SSH over Tailscale** (from any device on the tailnet, not just LAN): + +```bash +ssh pi@rpi-orcasound-bush-point +# or +ssh pi@$(tailscale ip -4) +``` + +**Local HLS segments** (confirms JACK/ffmpeg pipeline is healthy on this +hardware): + +```bash +docker compose exec streaming ls -lh /tmp//hls/ +``` + +**S3 upload**, once you're satisfied and have flipped `NO_UPLOAD=false` +in `.env` and restarted (`docker compose up -d` picks up the change): + +```bash +aws s3 ls s3://audio-orcasound-net//hls/ --human-readable +``` + +Confirm the objects land under the **new** `NODE_NAME` prefix, not the +original node's. + +--- + +## Troubleshooting + +**Original node dropped offline right as the new one came up.** +Classic Tailscale identity collision — the state file wasn't cleared +before `tailscale up`. Re-run Step 4's `rm`/restart sequence on the +*new* Pi. + +**Both Pis show the same hostname in the Tailscale admin console.** +`--hostname` wasn't picked up, or you renamed only one of LAN hostname +/ Tailscale hostname. Rename directly in the admin console as a +one-off fix, then align `raspi-config` to match. + +**New node's segments aren't showing up in S3 under the expected +prefix.** `.env` still has the old `NODE_NAME` — check `docker compose +exec streaming env | grep NODE_NAME` to see what the *running* +container actually has (stale containers keep old env until recreated +with `docker compose up -d`). + +**SSH warns about a host key mismatch when connecting to the new +Tailscale hostname.** Either Step 2's host-key regeneration was skipped, +or your local `~/.ssh/known_hosts` cached the source Pi's key under a +name now reused by the clone. Remove the stale entry: +`ssh-keygen -R rpi-orcasound-bush-point`. diff --git a/node_2026/README.md b/node_2026/README.md new file mode 100644 index 0000000..5fa3305 --- /dev/null +++ b/node_2026/README.md @@ -0,0 +1,402 @@ +# Orcasound Hydrophone Node + +| | | +|---|---| +| **Hardware** | Raspberry Pi 4 with Pisound HAT | +| **OS** | Raspberry Pi OS (Bookworm or Trixie) | +| **Container** | `orcasound/orcanode_val_docker` (built locally from `Dockerfile`) | + +## Overview + +This node captures audio from a Pisound HAT, segments it into HLS +(`.ts`) files using JACK + ffmpeg, and uploads them to S3. Everything +except Docker (and Tailscale, for remote access) runs inside the +container. Docker restarts the container automatically on crash or +reboot — no separate systemd service is needed. + +On startup `stream_sync.sh`: + +1. Waits for a sane system clock +2. Discovers the Pisound ALSA device +3. Starts `jackd` with the discovered hw address +4. Launches ffmpeg to capture from JACK and write: + - **hls-only**: HLS segments (`.ts`) + growing `live.m3u8` (all + segments for the day) + - **research**: same HLS output, plus lossless FLAC archive chunks + + Both modes embed absolute UTC timestamps (`EXT-X-PROGRAM-DATE-TIME`) + in the manifest so players and researchers can locate segments by + time. +5. Launches `upload_s3.py` to stream segments to S3 as they are written +6. Launches `catchup_s3.py` (`nice -n 10`) to recover segments missed + during any internet outage + +## Prerequisites + +- Raspberry Pi 4 with Pisound HAT installed and recognized by the OS +- Fresh Raspberry Pi OS image flashed to SD card +- SSH enabled, Pi connected to the internet +- AWS credentials with write access to the target S3 bucket +- A [Tailscale](https://tailscale.com) account (free tier is fine) — + used for remote SSH access once the node is deployed in the field + +--- + +## Step 1 — First Boot Configuration + +Flash your SD card with Raspberry Pi Imager and set: + +- Hostname (e.g. `rpi-orcasound-lab`) — pick this now; you'll reuse it + as the Tailscale machine name in Step 2 and as `NODE_NAME` in Step 4 +- SSH enabled +- Username: `pi` (or your preferred username) +- Password +- WiFi SSID and password (if not using ethernet) + +Boot the Pi and SSH in: + +```bash +ssh pi@ +``` + +--- + +## Step 2 — Install and Configure Tailscale + +Field-deployed nodes are usually headless and behind a NAT you don't +control, so plain SSH to a LAN IP stops working the moment the Pi +leaves your bench. Tailscale gives the node a stable address on your +private tailnet that works from anywhere, without port forwarding. + +Install and bring it up: + +```bash +curl -fsSL https://tailscale.com/install.sh | sh +sudo tailscale up +``` + +This prints an authentication URL. Open it in a browser and approve +the device against your tailnet. + +Once approved, rename the machine in the +[Tailscale admin console](https://login.tailscale.com/admin/machines) +to match the hostname you chose in Step 1 (e.g. `rpi-orcasound-lab`) — +keeping the LAN hostname, Tailscale name, and later `NODE_NAME` all in +sync makes the node much easier to identify once you have several +deployed. + +Confirm the node is up and get its tailnet address: + +```bash +tailscale status +tailscale ip -4 +``` + +The Pi should now appear as its own entry in the admin console with a +`100.x.x.x` address and a "last seen" time that keeps ticking forward. +From here on you can SSH in over Tailscale instead of the LAN IP — +useful for the rest of this setup, and essential once the node is +shipped to its deployment site: + +```bash +ssh pi@rpi-orcasound-lab +# or +ssh pi@$(tailscale ip -4) +``` + +> **Cloning this SD card for additional nodes?** A raw image clone +> copies Tailscale's node identity along with everything else, which +> causes the new Pi to collide with the source node on your tailnet. +> See [`CLONE_README.md`](CLONE_README.md) for the de-duplication +> steps required before a cloned card can come online safely. + +--- + +## Step 3 — Clone the Repository + +```bash +sudo apt-get install -y git +git clone https://github.com/orcasound/orcanode.git ~/orcanode +cd ~/orcanode/node_2026 +``` + +--- + +## Step 4 — Create the `.env` File + +The `.env` file holds node-specific config and AWS credentials. It is +never baked into the Docker image — Docker Compose injects it at +runtime. + +```bash +cp ~/orcanode/node_2026/.env.template ~/orcanode/node_2026/.env +nano ~/orcanode/node_2026/.env +``` + +Required variables: + +| Variable | Description | +|---|---| +| `NODE_NAME` | Unique name for this node — also used as the S3 path prefix. Match it to the Tailscale/LAN hostname from Steps 1–2 to keep node identity consistent everywhere. | +| `NODE_TYPE` | `hls-only` or `research` | +| `AUDIO_HW_ID` | Sound card name — verify with `aplay -l` | +| `SAMPLE_RATE` | `48000` | +| `CHANNELS` | `2` | +| `SEGMENT_DURATION` | HLS segment length in seconds | +| `FLAC_DURATION` | FLAC archive chunk length (research mode) | +| `NODE_LOOPBACK` | `true` to monitor audio on local output | +| `BUCKET_TYPE` | `prod`, `dev`, or `custom` | +| `AWS_ACCESS_KEY_ID` | Your AWS access key | +| `AWS_SECRET_ACCESS_KEY` | Your AWS secret key | +| `AWS_METADATA_SERVICE_TIMEOUT` | `5` | +| `AWS_METADATA_SERVICE_NUM_ATTEMPTS` | `0` | +| `REGION` | `us-west-2` | +| `SYSLOG_URL` | `syslog://syslog-a.logdna.com:37043` | +| `SYSLOG_STRUCTURED_DATA` | `logdna@48950 key="" tag="docker"` | +| `LC_ALL` | `C.UTF-8` | +| `NO_UPLOAD` | `false` — set `true` to test the pipeline without S3 | + +> **Note:** Set `NO_UPLOAD=true` during initial testing. Segments will +> accumulate locally in `/tmp//hls/` so you can verify the +> pipeline end-to-end before enabling live uploads. + +--- + +## Step 5 — Run Setup Script + +`setup.sh` installs Docker, fixes the Docker Hub IPv6 issue (common on +Pi OS Trixie), and adds the user to the `docker` and `audio` groups: + +```bash +cd ~/orcanode/node_2026 +bash setup.sh +``` + +The script reboots the Pi when complete. Wait for reboot, then SSH +back in (over Tailscale, if you're off the LAN by this point). + +> **Note:** `jackd`, `ffmpeg`, and Python run inside the Docker +> container — `setup.sh` does not install them on the host. + +--- + +## Step 6 — Build and Start the Container + +Build the image and start (first time only, or after code changes): + +```bash +cd ~/orcanode/node_2026 +docker compose up -d --build +``` + +After the first build, Docker manages the container automatically: + +- `restart: always` — restarts on crash without any action needed +- Docker enabled at boot — container starts on every reboot +- crontab (installed by `setup.sh`) restarts at midnight each night so + each calendar day gets its own S3 timestamp directory and a complete + `live.m3u8` covering only that day + +Watch the startup logs: + +```bash +docker compose logs -f +``` + +Healthy startup looks like: + +``` +Time looks sane: +Success! pisound found at index N. Using address: hw:N,0 +JACK is ready. +``` + +(then silence — ffmpeg and the uploaders run quietly) + +--- + +## Step 7 — Verify the Pipeline + +Check that HLS segments are being generated locally: + +```bash +docker compose exec streaming ls -lh /tmp//hls/ +``` + +Each `.ts` segment should be 150-300 KB. Watch them appear in real +time: + +```bash +docker compose exec streaming watch -n 1 'ls -lh /tmp//hls/*/' +``` + +In research mode, also check FLAC files are being written: + +```bash +docker compose exec streaming ls -lh /tmp//flac/ +``` + +Each `.flac` file covers `FLAC_DURATION` seconds of lossless audio. + +If `NO_UPLOAD=false`, verify segments are reaching S3: + +```bash +aws s3 ls s3://audio-orcasound-net//hls/ --human-readable +``` + +Segments are stored under a timestamp subdirectory, e.g.: + +``` +s3://audio-orcasound-net//hls//live000.ts +``` + +The live manifest (`live.m3u8`) grows throughout the day, accumulating +every segment since the last midnight restart. Inspect it to confirm +`program_date_time` tags: + +```bash +aws s3 cp s3://audio-orcasound-net//hls//live.m3u8 - +``` + +Check the upload log for RMS values (healthy signal = RMS > 100): + +```bash +docker compose logs -f +``` + +Finally, confirm the node is reachable remotely: from another device +on your tailnet, check the +[Tailscale admin console](https://login.tailscale.com/admin/machines) +for this node's entry and try `ssh pi@`. + +--- + +## Container Management + +```bash +docker compose up -d # start (after first build) +docker compose down # stop +docker compose restart # restart +docker compose logs -f # follow live logs +docker compose up -d --build # rebuild image and restart (after code changes) +``` + +Open a shell inside the running container: + +```bash +docker compose exec streaming /bin/bash +``` + +Check JACK port connections from inside the container: + +```bash +docker compose exec streaming jack_lsp -c +``` + +You should see `system:capture_1/2` connected to `ffjack:input_1/2`. + +--- + +## Internet Outage Recovery + +`upload_s3.py` leaves segments on disk when uploads fail. When +connectivity returns, `catchup_s3.py` finds stranded segments, +generates a VOD manifest (`catchup.m3u8`), and uploads everything at +low priority (2s between segments). + +**Disk guard:** if stranded segments exceed 500 MB, the oldest are +deleted first to protect the SD card. + +No configuration required — this runs automatically alongside +`upload_s3.py`. + +--- + +## Troubleshooting + +**Docker pull/push fails ("network is unreachable"):** +IPv6 issue on Pi OS Trixie. `setup.sh` fixes this automatically. If it +recurs, re-run `setup.sh` or manually add the Docker Hub IPv4 to +`/etc/hosts`: + +```bash +curl -4 -v https://registry-1.docker.io/v2/ 2>&1 | grep "Connected to" +echo " registry-1.docker.io" | sudo tee -a /etc/hosts +``` + +**JACK "Bus error" or "Cannot lock down memory":** +The container needs a larger `/dev/shm`. Verify `docker-compose.yml` +contains: + +```yaml +shm_size: '256m' +``` + +Rebuild and restart if you change it. + +**Audio device not found (pisound):** +Check the device is visible on the host: + +```bash +aplay -l +``` + +Verify `AUDIO_HW_ID` in `.env` matches the card name shown by +`aplay -l`. + +**Segments are silent (RMS near 0):** +JACK ports are not connected. Reconnect manually and restart: + +```bash +docker compose exec streaming jack_connect system:capture_1 ffjack:input_1 +docker compose exec streaming jack_connect system:capture_2 ffjack:input_2 +docker compose restart +``` + +**`.env` not loading:** +Verify no Windows line endings: + +```bash +file ~/orcanode/node_2026/.env +``` + +If it shows "CRLF", convert it: + +```bash +sed -i 's/\r//' ~/orcanode/node_2026/.env +``` + +**Node doesn't appear in the Tailscale admin console:** +Confirm `tailscaled` is actually running and the device authenticated +successfully: + +```bash +sudo systemctl status tailscaled +sudo tailscale up +``` + +If `tailscale up` reports it's already logged in but the node still +isn't listed, check you're looking at the correct tailnet (organization) +in the admin console — easy to mix up if you're a member of more than +one. + +**SSH over Tailscale hangs or refuses the connection:** +Confirm the Pi's Tailscale status is `Connected`, not `Idle` or +`NeedsLogin`: + +```bash +tailscale status +``` + +Also check the admin console for an ACL restricting SSH between +devices/tags — a permissive default tailnet allows it, but locked-down +tailnets need an explicit ACL rule. + +**Two nodes show up as the same machine, or one keeps dropping +offline right as another comes online:** +This is a cloned Tailscale identity — see the Troubleshooting section +of [`CLONE_README.md`](CLONE_README.md). + +--- + +For help, open an issue at: https://github.com/orcasound/orcanode diff --git a/node_2026/README.txt b/node_2026/README.txt deleted file mode 100644 index 71d7813..0000000 --- a/node_2026/README.txt +++ /dev/null @@ -1,244 +0,0 @@ -================================================================================ -ORCASOUND HYDROPHONE NODE -================================================================================ -Hardware: Raspberry Pi 4 with Pisound HAT -OS: Raspberry Pi OS (Bookworm or Trixie) -Container: orcasound/orcanode_val_docker (built locally from Dockerfile) -================================================================================ - -OVERVIEW --------- -This node captures audio from a Pisound HAT, segments it into HLS (.ts) files -using JACK + ffmpeg, and uploads them to S3. Everything except Docker runs -inside the container. Docker restarts the container automatically on crash or -reboot — no separate systemd service is needed. - -On startup stream_sync.sh: - 1. Waits for a sane system clock - 2. Discovers the Pisound ALSA device - 3. Starts jackd with the discovered hw address - 4. Launches ffmpeg to capture from JACK and write: - hls-only: HLS segments (.ts) + growing live.m3u8 (all segments for the day) - research: same HLS output, plus lossless FLAC archive chunks - Both modes embed absolute UTC timestamps (EXT-X-PROGRAM-DATE-TIME) - in the manifest so players and researchers can locate segments by time. - 5. Launches upload_s3.py to stream segments to S3 as they are written - 6. Launches catchup_s3.py (nice -n 10) to recover segments missed during - any internet outage - - -PREREQUISITES -------------- - - Raspberry Pi 4 with Pisound HAT installed and recognized by the OS - - Fresh Raspberry Pi OS image flashed to SD card - - SSH enabled, Pi connected to the internet - - AWS credentials with write access to the target S3 bucket - - --------------------------------------------------------------------------------- -STEP 1 - FIRST BOOT CONFIGURATION --------------------------------------------------------------------------------- - -Flash your SD card with Raspberry Pi Imager and set: - - Hostname (e.g. rpi-orcasound-lab) - - SSH enabled - - Username: pi (or your preferred username) - - Password - - WiFi SSID and password (if not using ethernet) - -Boot the Pi and SSH in: - ssh pi@ - - --------------------------------------------------------------------------------- -STEP 2 - CLONE THE REPOSITORY --------------------------------------------------------------------------------- - - sudo apt-get install -y git - git clone https://github.com/orcasound/orcanode.git ~/orcanode - cd ~/orcanode/node_val_docker - - --------------------------------------------------------------------------------- -STEP 3 - CREATE THE .ENV FILE --------------------------------------------------------------------------------- - -The .env file holds node-specific config and AWS credentials. It is never -baked into the Docker image — Docker Compose injects it at runtime. - - cp ~/orcanode/node_2026/.env.template ~/orcanode/node_2026/.env - nano ~/orcanode/node_2026/.env - -Required variables: - - NODE_NAME=rpi_orcasound_lab # unique name for this node - NODE_TYPE=hls-only # hls-only or research - AUDIO_HW_ID=pisound # sound card name — verify with: aplay -l - SAMPLE_RATE=48000 - CHANNELS=2 - SEGMENT_DURATION=10 # HLS segment length in seconds - FLAC_DURATION=30 # FLAC archive chunk length (research mode) - NODE_LOOPBACK=false # true to monitor audio on local output - BUCKET_TYPE=prod # prod, dev, or custom - AWS_ACCESS_KEY_ID= - AWS_SECRET_ACCESS_KEY= - AWS_METADATA_SERVICE_TIMEOUT=5 - AWS_METADATA_SERVICE_NUM_ATTEMPTS=0 - REGION=us-west-2 - SYSLOG_URL=syslog://syslog-a.logdna.com:37043 - SYSLOG_STRUCTURED_DATA='logdna@48950 key="" tag="docker"' - LC_ALL=C.UTF-8 - NO_UPLOAD=false # set to true to test pipeline without S3 - -NOTE: Set NO_UPLOAD=true during initial testing. Segments will accumulate -locally in /tmp//hls/ so you can verify the pipeline end-to-end -before enabling live uploads. - - --------------------------------------------------------------------------------- -STEP 4 - RUN SETUP SCRIPT --------------------------------------------------------------------------------- - -setup.sh installs Docker, fixes the Docker Hub IPv6 issue (common on Pi OS -Trixie), and adds the user to the docker and audio groups: - - cd ~/orcanode/node_val_docker - bash setup.sh - -The script reboots the Pi when complete. Wait for reboot, then SSH back in. - -Note: jackd, ffmpeg, and Python run inside the Docker container — setup.sh -does not install them on the host. - - --------------------------------------------------------------------------------- -STEP 5 - BUILD AND START THE CONTAINER --------------------------------------------------------------------------------- - -Build the image and start (first time only, or after code changes): - - cd ~/orcanode/node_val_docker - docker compose up -d --build - -After the first build, Docker manages the container automatically: - - restart: always — restarts on crash without any action needed - - Docker enabled at boot — container starts on every reboot - - crontab (installed by setup.sh) restarts at midnight each night so - each calendar day gets its own S3 timestamp directory and a complete - live.m3u8 covering only that day - -Watch the startup logs: - - docker compose logs -f - -Healthy startup looks like: - Time looks sane: - Success! pisound found at index N. Using address: hw:N,0 - JACK is ready. - (then silence — ffmpeg and the uploaders run quietly) - - --------------------------------------------------------------------------------- -STEP 6 - VERIFY THE PIPELINE --------------------------------------------------------------------------------- - -Check that HLS segments are being generated locally: - - docker compose exec streaming ls -lh /tmp//hls/ - -Each .ts segment should be 150-300 KB. Watch them appear in real time: - - docker compose exec streaming watch -n 1 'ls -lh /tmp//hls/*/' - -In research mode, also check FLAC files are being written: - - docker compose exec streaming ls -lh /tmp//flac/ - -Each .flac file covers FLAC_DURATION seconds of lossless audio. - -If NO_UPLOAD=false, verify segments are reaching S3: - - aws s3 ls s3://audio-orcasound-net//hls/ --human-readable - -Segments are stored under a timestamp subdirectory, e.g.: - s3://audio-orcasound-net//hls//live000.ts - -The live manifest (live.m3u8) grows throughout the day, accumulating every -segment since the last midnight restart. Inspect it to confirm program_date_time -tags: - - aws s3 cp s3://audio-orcasound-net//hls//live.m3u8 - - -Check the upload log for RMS values (healthy signal = RMS > 100): - - docker compose logs -f - - --------------------------------------------------------------------------------- -CONTAINER MANAGEMENT --------------------------------------------------------------------------------- - - docker compose up -d # start (after first build) - docker compose down # stop - docker compose restart # restart - docker compose logs -f # follow live logs - docker compose up -d --build # rebuild image and restart (after code changes) - -Open a shell inside the running container: - docker compose exec streaming /bin/bash - -Check JACK port connections from inside the container: - docker compose exec streaming jack_lsp -c - -You should see system:capture_1/2 connected to ffjack:input_1/2. - - --------------------------------------------------------------------------------- -INTERNET OUTAGE RECOVERY --------------------------------------------------------------------------------- - -upload_s3.py leaves segments on disk when uploads fail. When connectivity -returns, catchup_s3.py finds stranded segments, generates a VOD manifest -(catchup.m3u8), and uploads everything at low priority (2s between segments). - -Disk guard: if stranded segments exceed 500 MB, the oldest are deleted first -to protect the SD card. - -No configuration required — this runs automatically alongside upload_s3.py. - - --------------------------------------------------------------------------------- -TROUBLESHOOTING --------------------------------------------------------------------------------- - -Docker pull/push fails ("network is unreachable"): - IPv6 issue on Pi OS Trixie. setup.sh fixes this automatically. If it - recurs, re-run setup.sh or manually add the Docker Hub IPv4 to /etc/hosts: - curl -4 -v https://registry-1.docker.io/v2/ 2>&1 | grep "Connected to" - echo " registry-1.docker.io" | sudo tee -a /etc/hosts - -JACK "Bus error" or "Cannot lock down memory": - The container needs a larger /dev/shm. Verify docker-compose.yml contains: - shm_size: '256m' - Rebuild and restart if you change it. - -Audio device not found (pisound): - Check the device is visible on the host: - aplay -l - Verify AUDIO_HW_ID in .env matches the card name shown by aplay -l. - -Segments are silent (RMS near 0): - JACK ports are not connected. Reconnect manually and restart: - docker compose exec streaming jack_connect system:capture_1 ffjack:input_1 - docker compose exec streaming jack_connect system:capture_2 ffjack:input_2 - docker compose restart - -.env not loading: - Verify no Windows line endings: - file ~/orcanode/node_val_docker/.env - If it shows "CRLF", convert it: - sed -i 's/\r//' ~/orcanode/node_val_docker/.env - -================================================================================ -For help, open an issue at: https://github.com/orcasound/orcanode -================================================================================ From 658b5f79f16aeed82f601ca3497b12a87cc7dd24 Mon Sep 17 00:00:00 2001 From: val Date: Fri, 7 Aug 2026 11:36:10 -0700 Subject: [PATCH 7/9] node_2026: fix setup.sh PROJECT_DIR, document Ethernet setup, drop cloning section from main README - setup.sh: PROJECT_DIR was hardcoded to the old node_val_docker directory name, silently breaking the midnight-restart cron job (cd into a nonexistent path short-circuits the restart via &&) - README.md Step 1: document the Ethernet path explicitly (recommended over WiFi for field reliability), including mDNS discovery and an optional WiFi-radio-disable step - README.md: remove the SD-card cloning callout/troubleshooting entries pointing at CLONE_README.md, and clean up formatting around the Tailscale ACL tagging note in Step 2 Co-Authored-By: Claude Sonnet 5 --- node_2026/README.md | 52 +++++++++++++++++++++++++++++++++------------ node_2026/setup.sh | 2 +- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/node_2026/README.md b/node_2026/README.md index 5fa3305..f5d27ee 100644 --- a/node_2026/README.md +++ b/node_2026/README.md @@ -51,9 +51,40 @@ Flash your SD card with Raspberry Pi Imager and set: - SSH enabled - Username: `pi` (or your preferred username) - Password -- WiFi SSID and password (if not using ethernet) +- WiFi SSID and password — **only if using WiFi.** Ethernet is + recommended: it's more reliable for an unattended field node, and + skips WiFi setup entirely. If you're wiring Ethernet, leave the + Wireless LAN tab blank. -Boot the Pi and SSH in: +**Using Ethernet:** plug the cable into the Pi and your router/switch +*before* first power-on, so DHCP negotiation happens cleanly at boot. +Then boot the Pi and find it — no monitor needed: + +```bash +ping rpi-orcasound-lab.local # mDNS/Avahi, on by default +ssh pi@rpi-orcasound-lab.local +``` + +If `.local` doesn't resolve from your machine, check your router's +DHCP client list for the hostname/IP instead. A DHCP reservation (by +MAC address) is worth setting up once you see the IP, so it stays +consistent for direct LAN troubleshooting later — day-to-day remote +access will go through Tailscale instead (Step 2), so this is a +convenience, not a requirement. + +**Optional — disable the onboard WiFi radio.** If this Pi will only +ever use Ethernet, disabling WiFi entirely avoids it hunting for +networks, logging noise, or drawing power for nothing: + +```bash +echo "dtoverlay=disable-wifi" | sudo tee -a /boot/firmware/config.txt +sudo reboot +``` + +(`/boot/firmware/config.txt` is the Bookworm/Trixie path — not the +older `/boot/config.txt`.) + +**Using WiFi instead:** boot the Pi and SSH in directly: ```bash ssh pi@ @@ -94,6 +125,12 @@ tailscale ip -4 The Pi should now appear as its own entry in the admin console with a `100.x.x.x` address and a "last seen" time that keeps ticking forward. + +In the admin console, click the three dots next to the new machine and +choose **Edit ACL tags** to tag it with its role (e.g. `research`, +`production`, `veirs`) — useful once you have several nodes and want +to filter or write ACLs by tag. + From here on you can SSH in over Tailscale instead of the LAN IP — useful for the rest of this setup, and essential once the node is shipped to its deployment site: @@ -104,12 +141,6 @@ ssh pi@rpi-orcasound-lab ssh pi@$(tailscale ip -4) ``` -> **Cloning this SD card for additional nodes?** A raw image clone -> copies Tailscale's node identity along with everything else, which -> causes the new Pi to collide with the source node on your tailnet. -> See [`CLONE_README.md`](CLONE_README.md) for the de-duplication -> steps required before a cloned card can come online safely. - --- ## Step 3 — Clone the Repository @@ -392,11 +423,6 @@ Also check the admin console for an ACL restricting SSH between devices/tags — a permissive default tailnet allows it, but locked-down tailnets need an explicit ACL rule. -**Two nodes show up as the same machine, or one keeps dropping -offline right as another comes online:** -This is a cloned Tailscale identity — see the Troubleshooting section -of [`CLONE_README.md`](CLONE_README.md). - --- For help, open an issue at: https://github.com/orcasound/orcanode diff --git a/node_2026/setup.sh b/node_2026/setup.sh index abbc9cb..d12807f 100644 --- a/node_2026/setup.sh +++ b/node_2026/setup.sh @@ -7,7 +7,7 @@ set -e # Exit immediately if any command fails # Detect the actual user (works whether run as root or with sudo) REAL_USER=${SUDO_USER:-$USER} REAL_HOME=$(eval echo "~$REAL_USER") -PROJECT_DIR="$REAL_HOME/orcanode/node_val_docker" +PROJECT_DIR="$REAL_HOME/orcanode/node_2026" echo "=== Hydrophone Node Setup (Docker) ===" echo "Starting at $(date)" From 7ae9edb7b1b123986171b6bdaf989649a32b9d4d Mon Sep 17 00:00:00 2001 From: val Date: Fri, 7 Aug 2026 13:21:35 -0700 Subject: [PATCH 8/9] node_2026: fix clone command to use node_2026 branch, add Tailscale SSH/operator setup - Step 3: clone -b node_2026 explicitly, since the branch isn't merged into main yet and cd ~/orcanode/node_2026 fails on a plain clone - Step 2: add tailscale set --operator and tailscale set --ssh so the local user can manage tailscale without sudo and Tailscale SSH is available; note --force-reauth for re-authorizing an existing node Co-Authored-By: Claude Sonnet 5 --- node_2026/README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/node_2026/README.md b/node_2026/README.md index f5d27ee..09c0b7a 100644 --- a/node_2026/README.md +++ b/node_2026/README.md @@ -99,11 +99,14 @@ control, so plain SSH to a LAN IP stops working the moment the Pi leaves your bench. Tailscale gives the node a stable address on your private tailnet that works from anywhere, without port forwarding. -Install and bring it up: +Install and bring it up: ( sudo tailscale up --force-reauth to re-authorize connection to tailscale) ```bash curl -fsSL https://tailscale.com/install.sh | sh -sudo tailscale up +sudo tailscale up # --force-reauth +sudo tailscale set --operator=$USER +tailscale set --ssh + ``` This prints an authentication URL. Open it in a browser and approve @@ -145,9 +148,13 @@ ssh pi@$(tailscale ip -4) ## Step 3 — Clone the Repository +This node currently lives on the `node_2026` branch, not yet merged +into `main` — clone that branch directly, or `cd ~/orcanode/node_2026` +will fail with no such directory: + ```bash sudo apt-get install -y git -git clone https://github.com/orcasound/orcanode.git ~/orcanode +git clone -b node_2026 https://github.com/orcasound/orcanode.git ~/orcanode cd ~/orcanode/node_2026 ``` From 50102733471f832694df0933c09aae194722b58b Mon Sep 17 00:00:00 2001 From: val Date: Fri, 7 Aug 2026 21:51:38 -0700 Subject: [PATCH 9/9] node_2026: replace syslog/LogDNA config with Mezmo HTTP ingestion, delete leaked key - Add logdna_handler.py: an optional logging.Handler that forwards records to Mezmo's (formerly LogDNA) HTTPS ingestion API. No-op unless LOGDNA_INGESTION_KEY is set. Delivery failures log locally instead of raising, so a LogDNA outage can't break the uploader. - Wire it into upload_s3.py (WARNING+) and catchup_s3.py (INFO+, since catch-up activity itself signals a connectivity issue worth centralizing). - SYSLOG_URL/SYSLOG_STRUCTURED_DATA are gone: Docker's built-in syslog driver can't inject the RFC5424 structured-data block LogDNA's classic syslog ingestion required, so those vars were dead configuration nothing ever read. Replaced with one LOGDNA_INGESTION_KEY var across .env.template, README.md, and CLONE_README.md. - Delete the stale duplicate .env_template (underscore) which still had a real LogDNA ingestion key committed in plain text; the canonical .env.template (dot) was already properly sanitized and is now the only template. - README.md: document how to view forwarded logs in Mezmo (Host/App/ Level filters) and how to confirm delivery via docker compose logs. Co-Authored-By: Claude Sonnet 5 --- node_2026/.env.template | 3 +- node_2026/.env_template | 20 ------------ node_2026/CLONE_README.md | 2 +- node_2026/README.md | 51 ++++++++++++++++++++++++++++-- node_2026/catchup_s3.py | 6 ++++ node_2026/logdna_handler.py | 63 +++++++++++++++++++++++++++++++++++++ node_2026/upload_s3.py | 3 ++ 7 files changed, 123 insertions(+), 25 deletions(-) delete mode 100644 node_2026/.env_template create mode 100644 node_2026/logdna_handler.py diff --git a/node_2026/.env.template b/node_2026/.env.template index cbc32d5..8229a65 100644 --- a/node_2026/.env.template +++ b/node_2026/.env.template @@ -12,8 +12,7 @@ NODE_LOOPBACK=false SAMPLE_RATE=48000 AUDIO_HW_ID=pisound CHANNELS=2 -SYSLOG_URL=syslog://syslog-a.logdna.com:37043 -SYSLOG_STRUCTURED_DATA='logdna@48950 key="" tag="docker"' +#LOGDNA_INGESTION_KEY= FLAC_DURATION=30 SEGMENT_DURATION=10 LC_ALL=C.UTF-8 diff --git a/node_2026/.env_template b/node_2026/.env_template deleted file mode 100644 index ce872e6..0000000 --- a/node_2026/.env_template +++ /dev/null @@ -1,20 +0,0 @@ -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= -AWS_METADATA_SERVICE_TIMEOUT=5 -AWS_METADATA_SERVICE_NUM_ATTEMPTS=0 -REGION=us-west-2 -#BUCKET_TYPE=dev -BUCKET_TYPE=prod -#NODE_TYPE=research -NODE_TYPE=hls-only -NODE_NAME=rpi_orcasound_lab -NODE_LOOPBACK=true -SAMPLE_RATE=48000 -AUDIO_HW_ID=pisound -CHANNELS=2 -SYSLOG_URL=syslog://syslog-a.logdna.com:37043 -SYSLOG_STRUCTURED_DATA='logdna@48950 key="313dbd82f35ccbe462e6e3483984f464" tag="docker"' -FLAC_DURATION=30 -SEGMENT_DURATION=10 -LC_ALL=C.UTF-8 -NO_UPLOAD=true diff --git a/node_2026/CLONE_README.md b/node_2026/CLONE_README.md index 0f7311e..d2f3afa 100644 --- a/node_2026/CLONE_README.md +++ b/node_2026/CLONE_README.md @@ -132,7 +132,7 @@ nano .env | `BUCKET_TYPE` | Usually unchanged | Keep `prod` unless this new node is a test/dev deployment. | | `NO_UPLOAD` | Set `true` temporarily | Recommended for first boot — verify segments generate locally before enabling live S3 upload with a brand-new `NODE_NAME`. Flip to `false` once verified (see Step 6). | | `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` | Usually unchanged | Same bucket, same credentials — unless this node should log in under a separate IAM identity. | -| `SYSLOG_STRUCTURED_DATA` | Usually unchanged | Shared LogDNA ingestion key; logs from all nodes land in the same place, distinguished by `NODE_NAME` in the log content. | +| `LOGDNA_INGESTION_KEY` | Usually unchanged | Shared Mezmo/LogDNA ingestion key, if set; logs from all nodes land in the same place, distinguished by the `hostname` each node sends (its `NODE_NAME`). | | Everything else (`SAMPLE_RATE`, `CHANNELS`, `SEGMENT_DURATION`, `FLAC_DURATION`, `REGION`, `LC_ALL`) | Usually unchanged | Hardware/format constants, not node-specific. | Do **not** rebuild the Docker image for a `.env` change — `.env` is diff --git a/node_2026/README.md b/node_2026/README.md index 09c0b7a..a4a6ef0 100644 --- a/node_2026/README.md +++ b/node_2026/README.md @@ -189,8 +189,7 @@ Required variables: | `AWS_METADATA_SERVICE_TIMEOUT` | `5` | | `AWS_METADATA_SERVICE_NUM_ATTEMPTS` | `0` | | `REGION` | `us-west-2` | -| `SYSLOG_URL` | `syslog://syslog-a.logdna.com:37043` | -| `SYSLOG_STRUCTURED_DATA` | `logdna@48950 key="" tag="docker"` | +| `LOGDNA_INGESTION_KEY` | Optional. Set to forward `upload_s3.py`/`catchup_s3.py` logs to Mezmo (formerly LogDNA) — warnings/errors from the uploader, and catch-up activity after an outage. Leave unset to skip centralized logging entirely; nothing else depends on it. | | `LC_ALL` | `C.UTF-8` | | `NO_UPLOAD` | `false` — set `true` to test the pipeline without S3 | @@ -350,6 +349,54 @@ No configuration required — this runs automatically alongside --- +## Centralized Logging (Mezmo / LogDNA) + +If `LOGDNA_INGESTION_KEY` is set in `.env` (see Step 4), `upload_s3.py` +and `catchup_s3.py` forward selected log lines to +[Mezmo](https://app.mezmo.com/) (formerly LogDNA) over HTTPS, so you +can check on a node's health without SSHing in. This is optional — +leave the key unset and nothing changes. + +**What gets sent:** + +| Source | Minimum level forwarded | Typical content | +|---|---|---| +| `upload_s3.py` | `WARNING` | Low-RMS warnings (possible silence/bad capture), S3 upload failures | +| `catchup_s3.py` | `INFO` | Stranded segments found after an outage, disk-guard deletions, catch-up progress | + +Routine per-segment activity (every successful upload, RMS values on a +healthy signal) stays local-only by design, to avoid flooding a +rate-limited API — check those with `docker compose logs -f` instead. +Boot-time output (`stream_sync.sh`, JACK, ffmpeg) isn't sent to Mezmo +either, since it never passes through Python logging; that's local-only +too, same command. + +**To view the logs:** + +1. Log into [app.mezmo.com](https://app.mezmo.com/) with the account + tied to your ingestion key. +2. Each node reports under its `NODE_NAME` as the **Host** — use the + Host filter (left sidebar, or `host:` in the search bar) to narrow + to one node. This is why keeping `NODE_NAME` unique per node + (Step 4) matters here too. +3. Use the **App** filter (or `app:` in the search bar) to separate + `upload_s3` from `catchup_s3` events. +4. Use the **Level** filter to jump straight to `WARN`/`ERROR` — that's + the fastest way to spot a node that's gone silent or lost its audio + signal without reading through everything. + +If a node's logs aren't showing up in Mezmo at all, check the +container's own log first — the handler logs a local warning +(`LogDNA delivery failed: ...`) whenever it can't reach Mezmo, so +`docker compose logs -f` will tell you if the key is wrong or the node +has no route out: + +```bash +docker compose logs -f | grep -i logdna +``` + +--- + ## Troubleshooting **Docker pull/push fails ("network is unreachable"):** diff --git a/node_2026/catchup_s3.py b/node_2026/catchup_s3.py index 6590898..23d091e 100644 --- a/node_2026/catchup_s3.py +++ b/node_2026/catchup_s3.py @@ -20,6 +20,8 @@ import urllib.request import boto3 +from logdna_handler import attach_logdna_handler + NODE = os.environ["NODE_NAME"] SEGMENT_DURATION = int(os.environ.get("SEGMENT_DURATION", "10").strip()) BASEPATH = os.path.join("/tmp", NODE) @@ -47,6 +49,10 @@ handler = logging.StreamHandler(sys.stdout) handler.setFormatter(logging.Formatter("catchup.%(funcName)s: %(message)s")) log.addHandler(handler) +# INFO here (vs upload_s3.py's default WARNING) — catch-up activity itself +# (stranded segments found, disk guard deletions) is worth centralizing, +# since it signals the node had connectivity trouble. +attach_logdna_handler(log, NODE, app="catchup_s3", level=logging.INFO) def is_connected(): diff --git a/node_2026/logdna_handler.py b/node_2026/logdna_handler.py new file mode 100644 index 0000000..3d3f3ad --- /dev/null +++ b/node_2026/logdna_handler.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Optional logging.Handler that forwards records to Mezmo (formerly LogDNA) +over its HTTPS ingestion API. Enabled by setting LOGDNA_INGESTION_KEY in .env; +a no-op if that variable is unset, so it never affects nodes that don't use it. + +Delivery failures are logged locally rather than raised, so a LogDNA/network +outage never blocks the uploader itself. +""" + +import base64 +import json +import logging +import os +import urllib.error +import urllib.request + +INGEST_URL = "https://logs.mezmo.com/logs/ingest" +REQUEST_TIMEOUT = 5 # seconds + + +class LogDNAHandler(logging.Handler): + def __init__(self, ingestion_key, hostname, app): + super().__init__() + self.hostname = hostname + self.app = app + auth = base64.b64encode(f"{ingestion_key}:".encode()).decode() + self._headers = { + "Content-Type": "application/json; charset=UTF-8", + "Authorization": f"Basic {auth}", + } + self._fallback = logging.getLogger("logdna_handler") + + def emit(self, record): + try: + payload = json.dumps({ + "lines": [{ + "line": self.format(record), + "app": self.app, + "level": record.levelname, + "timestamp": int(record.created * 1000), + }] + }).encode("utf-8") + url = f"{INGEST_URL}?hostname={self.hostname}" + req = urllib.request.Request(url, data=payload, headers=self._headers, method="POST") + urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT) + except (urllib.error.URLError, OSError) as e: + self._fallback.warning(f"LogDNA delivery failed: {e}") + + +def attach_logdna_handler(log, node_name, app, level=logging.WARNING): + """Attach a LogDNA handler to `log` if LOGDNA_INGESTION_KEY is set in the + environment. No-op otherwise. `level` controls the minimum severity + forwarded — defaults to WARNING to avoid shipping routine per-segment + debug chatter to a paid, rate-limited API. + """ + key = os.environ.get("LOGDNA_INGESTION_KEY") + if not key: + return + handler = LogDNAHandler(key, node_name, app) + handler.setLevel(level) + handler.setFormatter(logging.Formatter('%(module)s.%(funcName)s: %(message)s')) + log.addHandler(handler) + log.info("LogDNA logging enabled (level=%s)", logging.getLevelName(level)) diff --git a/node_2026/upload_s3.py b/node_2026/upload_s3.py index 965b8e2..74132d4 100644 --- a/node_2026/upload_s3.py +++ b/node_2026/upload_s3.py @@ -11,6 +11,8 @@ import os import sys +from logdna_handler import attach_logdna_handler + NODE = os.environ["NODE_NAME"] BASEPATH = os.path.join("/tmp", NODE) PATH = os.path.join(BASEPATH, "hls") @@ -24,6 +26,7 @@ formatter = logging.Formatter('%(module)s.%(funcName)s: %(message)s') handler.setFormatter(formatter) log.addHandler(handler) +attach_logdna_handler(log, NODE, app="upload_s3") BUCKET = "" if "BUCKET_TYPE" in os.environ: