Github Actions Docker images auto build

.github/workflows/docker-image.yml

@@ -0,0 +1,129 @@
+name: Docker Image CI
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+
+  docker-build-db:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner}}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: lowercase github.repository
+        run: echo "IMAGE_NAME=${GITHUB_REPOSITORY@L}" >> ${GITHUB_ENV}
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          file: trackdirect-db.dockerfile
+          push: true
+          tags: |
+            ghcr.io/${{ env.IMAGE_NAME }}-db:latest
+            ghcr.io/${{ env.IMAGE_NAME }}-db:${{ github.RUN_ID }}
+
+  docker-build-apache:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner}}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: lowercase github.repository
+        run: echo "IMAGE_NAME=${GITHUB_REPOSITORY@L}" >> ${GITHUB_ENV}
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          file: trackdirect-apache.dockerfile
+          push: true
+          tags: |
+            ghcr.io/${{ env.IMAGE_NAME }}-apache:latest
+            ghcr.io/${{ env.IMAGE_NAME }}-apache:${{ github.RUN_ID }}
+
+  docker-build-python:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner}}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: lowercase github.repository
+        run: echo "IMAGE_NAME=${GITHUB_REPOSITORY@L}" >> ${GITHUB_ENV}
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          file: trackdirect-python.dockerfile
+          push: true
+          tags: |
+            ghcr.io/${{ env.IMAGE_NAME }}-python:latest
+            ghcr.io/${{ env.IMAGE_NAME }}-python:${{ github.RUN_ID }}
+
+  docker-build-aprsc:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner}}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: lowercase github.repository
+        run: echo "IMAGE_NAME=${GITHUB_REPOSITORY@L}" >> ${GITHUB_ENV}
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          file: trackdirect-aprsc.dockerfile
+          push: true
+          tags: |
+            ghcr.io/${{ env.IMAGE_NAME }}-aprsc:latest
+            ghcr.io/${{ env.IMAGE_NAME }}-aprsc:${{ github.RUN_ID }}
+
+  docker-build-cron:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner}}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: lowercase github.repository
+        run: echo "IMAGE_NAME=${GITHUB_REPOSITORY@L}" >> ${GITHUB_ENV}
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          file: trackdirect-cron.dockerfile
+          push: true
+          tags: |
+            ghcr.io/${{ env.IMAGE_NAME }}-cron:latest
+            ghcr.io/${{ env.IMAGE_NAME }}-cron:${{ github.RUN_ID }}

crontab

@@ -1,2 +1,2 @@
 40 * * * * ~/trackdirect/server/scripts/remover.sh trackdirect.ini
-*/30 * * * * ~/trackdirect/server/scripts/ogn_devices_install.sh trackdirect db 5432 postgres
+*/30 * * * * ~/trackdirect/server/scripts/ogn_devices_install.sh trackdirect.ini

htdocs/public/index.php

@@ -45,7 +45,7 @@
                 <script src="https://cdnjs.cloudflare.com/ajax/libs/mapbox-gl-leaflet/0.0.15/leaflet-mapbox-gl.min.js"></script>
             <?php endif; ?>
 
-            <script src="https://cdnjs.cloudflare.com/ajax/libs/leaflet-providers/1.11.0/leaflet-providers.min.js" integrity="sha512-TO+Wd5hbpDsACTmvzSqAZL83jMQCXGRFNoS4WZxcxrlJBTdgMYaT7g5uX49C5+Kbuxzlg2A+TFJ6UqdsXuOKLw==" crossorigin="anonymous"></script>
+            <script src="https://cdnjs.cloudflare.com/ajax/libs/leaflet-providers/1.13.0/leaflet-providers.min.js" integrity="sha512-5EYsvqNbFZ8HX60keFbe56Wr0Mq5J1RrA0KdVcfGDhnjnzIRsDrT/S3cxdzpVN2NGxAB9omgqnlh4/06TvWCMw==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
             <script src="https://cdnjs.cloudflare.com/ajax/libs/leaflet.heat/0.2.0/leaflet-heat.js" integrity="sha512-KhIBJeCI4oTEeqOmRi2gDJ7m+JARImhUYgXWiOTIp9qqySpFUAJs09erGKem4E5IPuxxSTjavuurvBitBmwE0w==" crossorigin="anonymous"></script>
             <script src="https://cdnjs.cloudflare.com/ajax/libs/OverlappingMarkerSpiderfier-Leaflet/0.2.6/oms.min.js" integrity="sha512-V8RRDnS4BZXrat3GIpnWx+XNYBHQGdK6nKOzMpX4R0hz9SPWt7fltGmmyGzUkVFZUQODO1rE+SWYJJkw3SYMhg==" crossorigin="anonymous"></script>
         <?php endif; ?>

server/bin/remover.py

@@ -27,8 +27,8 @@ def validate_config_file(config_file):
         print("\nUsage: script.py [config.ini]")
         sys.exit()
 
-def drop_table_if_exists(cursor, table_name, logger):
-    if DatabaseObjectFinder.check_table_exists(table_name):
+def drop_table_if_exists(cursor, dbobjfinder, table_name, logger):
+    if dbobjfinder.check_table_exists(table_name):
         cursor.execute(f"DROP TABLE {table_name}")
         logger.info(f"Dropped table {table_name}")
 
@@ -52,11 +52,21 @@ def main():
     max_days_to_save_weather_data = config.days_to_save_weather_data
     max_days_to_save_telemetry_data = config.days_to_save_telemetry_data
 
+    # Use the highest retention across all data types so drop loops cover everything
+    max_retention = max(
+        max_days_to_save_position_data,
+        max_days_to_save_weather_data,
+        max_days_to_save_telemetry_data
+    )
+    # Scan far enough ahead to handle long cron outages (up to ~1 year)
+    drop_lookahead = 365
+
     logger.info("Starting")
     logger.info(f"Saving position data for {max_days_to_save_position_data} days")
     logger.info(f"Saving station data for {max_days_to_save_station_data} days")
     logger.info(f"Saving weather data for {max_days_to_save_weather_data} days")
     logger.info(f"Saving telemetry data for {max_days_to_save_telemetry_data} days")
+    logger.info(f"Drop lookahead set to {drop_lookahead} days (max retention = {max_retention})")
 
     try:
         track_direct_db = DatabaseConnection()
@@ -69,56 +79,62 @@ def main():
         track_direct_db_object_finder = DatabaseObjectFinder(db)
         packet_repository = PacketRepository(db)
 
-        # Loop over the latest days and delete packets that are not needed anymore
-        for x in range(2, 16):
+        for x in range(2, max_days_to_save_position_data + 1):
             prev_day = datetime.date.today() - datetime.timedelta(x)
             prev_day_timestamp = int(prev_day.strftime("%s"))
-            prev_day_format = datetime.datetime.utcfromtimestamp(prev_day_timestamp).strftime('%Y%m%d')
+            prev_day_format = datetime.datetime.fromtimestamp(prev_day_timestamp, datetime.timezone.utc).strftime('%Y%m%d')
             packet_table = f"packet{prev_day_format}"
 
             if track_direct_db_object_finder.check_table_exists(packet_table):
-                deleted_rows = None
-                do_full_vacuum = False
-                while deleted_rows is None or deleted_rows >= 5000:
-                    sql = f"""DELETE FROM {packet_table} WHERE id IN (
-                              SELECT id FROM {packet_table} WHERE map_id NOT IN (1,12) LIMIT 5000)"""
-                    cursor.execute(sql)
-                    deleted_rows = cursor.rowcount
-                    logger.info(f"Deleted {deleted_rows} from {packet_table}")
-                    if deleted_rows > 0:
-                        do_full_vacuum = True
-                    time.sleep(0.5)
-                if do_full_vacuum:
-                    cursor.execute(f"VACUUM FULL {packet_table}_path")
-                    cursor.execute(f"REINDEX TABLE {packet_table}_path")
-                    cursor.execute(f"VACUUM FULL {packet_table}")
-                    cursor.execute(f"REINDEX TABLE {packet_table}")
-
-        # Drop packet_weather
-        for x in range(max_days_to_save_weather_data, max_days_to_save_weather_data + 100):
-            prev_day = datetime.date.today() - datetime.timedelta(x)
-            prev_day_format = prev_day.strftime('%Y%m%d')
-            packet_table = f"packet{prev_day_format}_weather"
-            drop_table_if_exists(cursor, packet_table, logger)
-
-        # Drop packet_telemetry
-        for x in range(max_days_to_save_telemetry_data, max_days_to_save_telemetry_data + 100):
+                # Count rows to delete first
+                cursor.execute(f"SELECT count(*) FROM {packet_table} WHERE map_id NOT IN (1,12)")
+                count = cursor.fetchone()[0]
+                if count == 0:
+                    logger.info(f"No stale rows in {packet_table}, skipping")
+                    continue
+
+                logger.info(f"Deleting {count} stale rows from {packet_table}")
+                cursor.execute(f"DELETE FROM {packet_table} WHERE map_id NOT IN (1,12)")
+                deleted_rows = cursor.rowcount
+                logger.info(f"Deleted {deleted_rows} from {packet_table}")
+
+                if deleted_rows > 0:
+                    logger.info(f"Cleanup deferred to autovacuum for {packet_table}")
+
+        scanned_days = 0
+        dropped_tables = 0
+        for x in range(max_retention, max_retention + drop_lookahead + 1):
+            scanned_days += 1
             prev_day = datetime.date.today() - datetime.timedelta(x)
             prev_day_format = prev_day.strftime('%Y%m%d')
-            packet_table = f"packet{prev_day_format}_telemetry"
-            drop_table_if_exists(cursor, packet_table, logger)
-
-        # Drop packets
-        for x in range(max_days_to_save_position_data, max_days_to_save_position_data + 100):
-            prev_day = datetime.date.today() - datetime.timedelta(x)
-            prev_day_format = prev_day.strftime('%Y%m%d')
-            packet_table = f"packet{prev_day_format}"
-
-            drop_table_if_exists(cursor, f"{packet_table}_ogn", logger)
-            drop_table_if_exists(cursor, f"{packet_table}_path", logger)
-            drop_table_if_exists(cursor, packet_table, logger)
+            base_table = f"packet{prev_day_format}"
+
+            # Drop weather sub-table (only if past weather retention)
+            if x >= max_days_to_save_weather_data:
+                if track_direct_db_object_finder.check_table_exists(f"{base_table}_weather"):
+                    dropped_tables += 1
+                drop_table_if_exists(cursor, track_direct_db_object_finder, f"{base_table}_weather", logger)
+
+            # Drop telemetry sub-table (only if past telemetry retention)
+            if x >= max_days_to_save_telemetry_data:
+                if track_direct_db_object_finder.check_table_exists(f"{base_table}_telemetry"):
+                    dropped_tables += 1
+                drop_table_if_exists(cursor, track_direct_db_object_finder, f"{base_table}_telemetry", logger)
+
+            # Drop position sub-tables and main table (only if past position retention)
+            if x >= max_days_to_save_position_data:
+                if track_direct_db_object_finder.check_table_exists(f"{base_table}_ogn"):
+                    dropped_tables += 1
+                drop_table_if_exists(cursor, track_direct_db_object_finder, f"{base_table}_ogn", logger)
+                if track_direct_db_object_finder.check_table_exists(f"{base_table}_path"):
+                    dropped_tables += 1
+                drop_table_if_exists(cursor, track_direct_db_object_finder, f"{base_table}_path", logger)
+                if track_direct_db_object_finder.check_table_exists(base_table):
+                    dropped_tables += 1
+                drop_table_if_exists(cursor, track_direct_db_object_finder, base_table, logger)
+
+        logger.info(f"Phase 2 complete: scanned {scanned_days} days, dropped {dropped_tables} tables")
 
-        # Delete old stations
         timestamp_limit = int(time.time()) - (60 * 60 * 24 * max_days_to_save_station_data)
         deleted_rows = 0
         sql = """SELECT station.id, station.latest_sender_id, station.name
@@ -148,7 +164,7 @@ def main():
             logger.info(f"Trying to delete station {record['name']} ({record['id']})")
             delete_cursor = db_no_auto_commit.cursor()
             try:
-                for table in ['station_telemetry_bits', 'station_telemetry_eqns', 'station_telemetry_param', 'station_telemetry_unit', 'station_city']:
+                for table in ['station_telemetry_bits', 'station_telemetry_eqns', 'station_telemetry_param', 'station_telemetry_unit']:
                     delete_cursor.execute(f"DELETE FROM {table} WHERE station_id = %s", (record["id"],))
 
                 delete_cursor.execute("DELETE FROM station WHERE id = %s", (record["id"],))
@@ -157,7 +173,6 @@ def main():
                 db_no_auto_commit.commit()
                 delete_cursor.close()
                 deleted_rows += 1
-                time.sleep(0.5)
             except Exception as e:
                 logger.error(e, exc_info=1)
                 db_no_auto_commit.rollback()
@@ -172,7 +187,6 @@ def main():
         cursor.execute("VACUUM ANALYZE sender")
         cursor.execute("REINDEX TABLE sender")
 
-        # Close DB connection
         cursor.close()
         db.close()
         logger.info("Done!")

server/scripts/ogn_devices_install.sh

@@ -2,25 +2,40 @@
 if [ $# -eq 0 ]
   then
     echo "No arguments supplied"
-    echo "$0 [dbname] [dbport] [dbuser]"
+    echo "$0 [config file]"
     exit
 fi
 
-DATABASE=$1
-HOST=$2
-PORT=$3
-USER=$4
+CONFIGFILE=$1
+if [[ "$CONFIGFILE" != /* ]]; then
+    CONFIGFILE="$HOME/trackdirect/config/$CONFIGFILE"
+fi
+
+if [ ! -f "$CONFIGFILE" ]; then
+    echo "Config file $CONFIGFILE does not exist"
+    exit
+fi
+
+# Parse database credentials from the .ini config file
+DB_HOST=$(grep -oP '^host\s*=\s*\K.*' "$CONFIGFILE" | head -1)
+DB_PORT=$(grep -oP '^port\s*=\s*\K.*' "$CONFIGFILE" | head -1)
+DB_NAME=$(grep -oP '^database\s*=\s*\K.*' "$CONFIGFILE" | head -1)
+DB_USER=$(grep -oP '^username\s*=\s*\K.*' "$CONFIGFILE" | head -1)
+DB_PASS=$(grep -oP '^password\s*=\s*\K.*' "$CONFIGFILE" | head -1)
+
+# Export password so psql doesn't need .pgpass
+export PGPASSWORD="$DB_PASS"
 
 pushd `dirname $0` > /dev/null
 SCRIPTPATH=`pwd -P`
 popd > /dev/null
 
 # Create dir and remove old stuff (keep zip-file since it may be equal to latest)
 mkdir -p $SCRIPTPATH/ogndevices
-mkdir -p $SCRIPTPATH/ogndevices/${DATABASE}
-rm $SCRIPTPATH/ogndevices/${DATABASE}/*.csv
-rm $SCRIPTPATH/ogndevices/${DATABASE}/*.txt
-cd $SCRIPTPATH/ogndevices/${DATABASE}
+mkdir -p $SCRIPTPATH/ogndevices/${DB_NAME}
+rm $SCRIPTPATH/ogndevices/${DB_NAME}/*.csv
+rm $SCRIPTPATH/ogndevices/${DB_NAME}/*.txt
+cd $SCRIPTPATH/ogndevices/${DB_NAME}
 
 # Download latest csv file (but only if newer)
 wget -N http://ddb.glidernet.org/download/?t=1 -O ogndevices.csv
@@ -34,8 +49,8 @@ else
 # Remove comments in file
 sed '/^#/ d' < ogndevices.csv > ogndevices2.csv
 
-# Load file into database (assumes .pgpass is correctly set)
-psql -h $HOST -p $PORT $DATABASE -U $USER << EOF
+# Load file into database (password comes from PGPASSWORD env var)
+psql -h $DB_HOST -p $DB_PORT $DB_NAME -U $DB_USER << EOF
 
 create table if not exists ogn_device (
     "device_type" text not null,
@@ -52,7 +67,7 @@ begin transaction;
 
 drop index if exists ogn_device_device_id_idx;
 truncate ogn_device;
-\copy ogn_device from '$SCRIPTPATH/ogndevices/$DATABASE/ogndevices2.csv' DELIMITERS ',' CSV QUOTE '''';
+\copy ogn_device from '$SCRIPTPATH/ogndevices/$DB_NAME/ogndevices2.csv' DELIMITERS ',' CSV QUOTE '''';
 create index ogn_device_device_id_idx on ogn_device(device_id);
 
 insert into ogn_device(device_type, device_id, aircraft_model, registration, cn, tracked, identified, ddb_aircraft_type) values ('F', '3FEF6F', '', '', '', 'N', 'N', 1);
@@ -62,4 +77,4 @@ EOF
 
 fi
 
-exit 0
+exit 0

trackdirect-apache.dockerfile

@@ -1,4 +1,4 @@
-FROM php:8.2-apache
+FROM php:8.4.8-apache
 
 RUN apt-get update && apt-get install -y \
   git \

trackdirect-aprsc.dockerfile

@@ -4,7 +4,7 @@ RUN apt-get update && apt-get install -y \
   gnupg \
   && rm -rf /var/lib/apt/lists/*
 
-RUN printf "deb http://aprsc-dist.he.fi/aprsc/apt noble main" >> /etc/apt/sources.list
+RUN printf "deb [trusted=yes] http://aprsc-dist.he.fi/aprsc/apt noble main" >> /etc/apt/sources.list
 RUN gpg --keyserver keyserver.ubuntu.com --recv C51AA22389B5B74C3896EF3CA72A581E657A2B8D
 RUN gpg --export C51AA22389B5B74C3896EF3CA72A581E657A2B8D | apt-key add -