From 1df8093c5ab44b4f5f95eca14b0057addc10c91f Mon Sep 17 00:00:00 2001 From: "Thomas Neep (Advanced Research Computing)" Date: Tue, 3 Mar 2026 13:33:29 +0000 Subject: [PATCH 1/7] Start adding infrastructure for building a container in CI The goal here is to be able to build an `agate` container. It should just include `agate` and be configurable (enough) so that we can run the container with the right settings. --- agate/.dockerignore | 39 ++++++++ agate/Dockerfile | 20 ++++ agate/core/production.py | 204 +++++++++++++++++++++++++++++++++++++++ agate/entrypoint.sh | 8 ++ agate/requirements.txt | 11 +++ 5 files changed, 282 insertions(+) create mode 100644 agate/.dockerignore create mode 100644 agate/Dockerfile create mode 100644 agate/core/production.py create mode 100755 agate/entrypoint.sh create mode 100644 agate/requirements.txt diff --git a/agate/.dockerignore b/agate/.dockerignore new file mode 100644 index 0000000..d211631 --- /dev/null +++ b/agate/.dockerignore @@ -0,0 +1,39 @@ +**/__pycache__/ +*.pyc +*.pyo +*.pyd + +venv/ +env/ +.venv/ + +.env +*.env + +.git/ +.gitignore +.gitattributes + +.DS_Store +Thumbs.db +.idea/ +.vscode/ + +**/*~ + +*.sqlite3 + +docker-compose.yml +Dockerfile + +staticfiles/ +static/ +media/ +media_root/ + +build/ +dist/ +*.egg-info/ + +coverage_html_report/ +*log diff --git a/agate/Dockerfile b/agate/Dockerfile new file mode 100644 index 0000000..2947a2d --- /dev/null +++ b/agate/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.12-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +ENV DJANGO_SETTINGS_MODULE=core.production + +WORKDIR /app + +COPY requirements.txt /app/ + +# Install requirements from the project +RUN pip install --upgrade pip && \ + pip install --no-cache-dir -r requirements.txt + +COPY . /app + +COPY entrypoint.sh /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["gunicorn", "core.wsgi:application", "--bind", "0.0.0.0:8000"] diff --git a/agate/core/production.py b/agate/core/production.py new file mode 100644 index 0000000..ae426e2 --- /dev/null +++ b/agate/core/production.py @@ -0,0 +1,204 @@ +import os +import sys +from empty_message_retrieval import EmptyMessageRetrieval + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +TEMPLATE_DIR = os.path.join(BASE_DIR, 'core/templates') + + +# Application definition + +INSTALLED_APPS = [ + # Django default apps + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'django.contrib.humanize', + # Custom apps + 'account', + 'agate', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + "corsheaders.middleware.CorsMiddleware", + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'core.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [TEMPLATE_DIR], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + 'libraries': { + 'settings_value': 'core.templatetags.settings_value', + } + }, + }, +] + +WSGI_APPLICATION = 'core.wsgi.application' + + +# Custom user model for authentication + +AUTH_USER_MODEL = 'account.User' + + +# Password validation + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization + +LANGUAGE_CODE = 'en-gb' + +TIME_ZONE = 'Europe/London' + +USE_I18N = False # Set to True if using internationalisation (translations) in your project + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) + +STATICFILES_DIRS = [os.path.join(BASE_DIR, "core/static")] +STATIC_URL = '/static/' +STATIC_ROOT = os.path.join(BASE_DIR, "static/") + + +# Media files (user uploaded content) + +MEDIA_URL = '/media/' +MEDIA_ROOT = os.path.join(BASE_DIR, 'media') + + +# Default primary key field type +# https://docs.djangoproject.com/en/latest/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + + +DEBUG = os.environ.get("DEBUG", False) + + +# Storages + +# Default STORAGES from Django documentation +# See: https://docs.djangoproject.com/en/4.2/ref/settings/#std-setting-STORAGES +STORAGES = { + "default": {"BACKEND": "django.core.files.storage.FileSystemStorage"}, + "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}, +} + +# Use ManifestStaticFilesStorage when not in debug mode +if not DEBUG: # NOQA + STORAGES['staticfiles'] = {"BACKEND": "django.contrib.staticfiles.storage.ManifestStaticFilesStorage"} + + +# Logging + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'filters': { + 'require_debug_false': { + '()': 'django.utils.log.RequireDebugFalse', + }, + 'require_debug_true': { + '()': 'django.utils.log.RequireDebugTrue', + }, + }, + 'formatters': { + 'verbose': { + 'format': '{levelname} {asctime} {module} {process:d} {thread:d} {message}', + 'style': '{', + }, + 'simple': { + 'format': '{levelname} {message}', + 'style': '{', + }, + }, + 'handlers': { + 'stream': { + 'class': 'logging.StreamHandler', + 'level': 'DEBUG' if DEBUG else 'INFO', # NOQA + 'formatter': 'verbose', + }, + 'file': { + 'level': 'DEBUG', + 'filters': ['require_debug_true'], + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': os.path.join(BASE_DIR, 'django.log'), + 'formatter': 'verbose', + 'maxBytes': 104857600, # 100*1024*1024 bytes (100MB) + }, + 'mail_admins': { + 'level': 'ERROR', + 'filters': ['require_debug_false'], + 'class': 'django.utils.log.AdminEmailHandler', + }, + }, + 'loggers': { + 'django': { + 'handlers': ['stream', 'file', 'mail_admins'], + 'level': 'DEBUG' if DEBUG else 'INFO', # NOQA + 'propagate': 'True', + }, + }, +} + +# We are required to set all these variables to run the app +SECRET_KEY = os.environ["SECRET_KEY"] + +ALLOWED_HOSTS = os.environ["ALLOWED_HOSTS"].split(",") + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'django-template.sqlite3'), + 'TEST': { + 'NAME': os.path.join(BASE_DIR, 'django-template_TEST.sqlite3'), + }, + } +} + +ONYX_DOMAIN = os.environ["ONYX_DOMAIN"] + +MESSAGE_RETRIEVAL = EmptyMessageRetrieval() + +LIMITED_PROJECT_LIST = os.environ.get( + "LIMITED_PROJECT_LIST", "synthscape", +).split(",") diff --git a/agate/entrypoint.sh b/agate/entrypoint.sh new file mode 100755 index 0000000..01ca90e --- /dev/null +++ b/agate/entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +echo "Applying database migrations..." +python manage.py migrate --noinput + +echo "Starting server" +exec "$@" diff --git a/agate/requirements.txt b/agate/requirements.txt new file mode 100644 index 0000000..bc68e28 --- /dev/null +++ b/agate/requirements.txt @@ -0,0 +1,11 @@ +# Please follow suitable version specifiers, as outlined in https://www.python.org/dev/peps/pep-0440/#version-specifiers +# For Django versioning we use ~= (e.g. Django~=4.2.0) which will keep the minor version up to date (e.g. the latest version prior to 4.3.0) + +Django~=4.2.0 +djangorestframework +django-cors-headers +psycopg2-binary +requests +apscheduler +varys-client +gunicorn From 47965b51bd4ee5aa2e99566a5203c10194940e50 Mon Sep 17 00:00:00 2001 From: "Thomas Neep (Advanced Research Computing)" Date: Tue, 3 Mar 2026 13:45:35 +0000 Subject: [PATCH 2/7] Remove sys from production.py settings --- agate/core/production.py | 1 - 1 file changed, 1 deletion(-) diff --git a/agate/core/production.py b/agate/core/production.py index ae426e2..1878c62 100644 --- a/agate/core/production.py +++ b/agate/core/production.py @@ -1,5 +1,4 @@ import os -import sys from empty_message_retrieval import EmptyMessageRetrieval # Build paths inside the project like this: os.path.join(BASE_DIR, ...) From 77869aefc8e91e4f0862e878b4c12436cc4d07be Mon Sep 17 00:00:00 2001 From: "Thomas Neep (Advanced Research Computing)" Date: Tue, 3 Mar 2026 14:18:12 +0000 Subject: [PATCH 3/7] Update production settings to use postgres and varys --- agate/core/production.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/agate/core/production.py b/agate/core/production.py index 1878c62..1f9f8ad 100644 --- a/agate/core/production.py +++ b/agate/core/production.py @@ -184,19 +184,31 @@ ALLOWED_HOSTS = os.environ["ALLOWED_HOSTS"].split(",") +# Set the database name below DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'django-template.sqlite3'), - 'TEST': { - 'NAME': os.path.join(BASE_DIR, 'django-template_TEST.sqlite3'), - }, + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": os.environ["DATABASE_NAME"], + "USER": os.environ["DATABASE_USER"], + "PASSWORD": os.environ["DATABASE_PASSWORD"], + "HOST": os.environ["DATABASE_HOST"], } } +CORS_ALLOWED_ORIGINS = os.environ.get("CORS_ALLOWED_ORIGINS", "").split(",") +CSRF_TRUSTED_ORIGINS = os.environ.get("CSRF_TRUSTED_ORIGINS", "").split(",") + ONYX_DOMAIN = os.environ["ONYX_DOMAIN"] -MESSAGE_RETRIEVAL = EmptyMessageRetrieval() +MESSAGE_RETRIEVAL = VarysMessageRetrieval( + queue_suffix="agate", + timeout=1, + config_path=os.path.join(BASE_DIR, "varys_config.cfg"), + profile="test", + logfile="varys.log", + log_level="DEBUG", + auto_acknowledge=False +) LIMITED_PROJECT_LIST = os.environ.get( "LIMITED_PROJECT_LIST", "synthscape", From ed7a3b56b690ea472d5b9d760e663ef20ba69b16 Mon Sep 17 00:00:00 2001 From: "Thomas Neep (Advanced Research Computing)" Date: Tue, 3 Mar 2026 14:18:53 +0000 Subject: [PATCH 4/7] Fix message import --- agate/core/production.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agate/core/production.py b/agate/core/production.py index 1f9f8ad..f09c8bb 100644 --- a/agate/core/production.py +++ b/agate/core/production.py @@ -1,5 +1,5 @@ import os -from empty_message_retrieval import EmptyMessageRetrieval +from varys_message_retrieval import VarysMessageRetrieval # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) From 954ecce3ffcabab36ef3e4d9d480b9daa82ece8c Mon Sep 17 00:00:00 2001 From: "Thomas Neep (Advanced Research Computing)" Date: Tue, 3 Mar 2026 14:30:09 +0000 Subject: [PATCH 5/7] Add varys_config to .dockerignore --- agate/.dockerignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/agate/.dockerignore b/agate/.dockerignore index d211631..1951104 100644 --- a/agate/.dockerignore +++ b/agate/.dockerignore @@ -37,3 +37,7 @@ dist/ coverage_html_report/ *log + +# The varys_config.cfg needs to be given when setting up the container +e.g. with a volume mount or some other technique +varys_config.cfg From d49993ceee01c834b9dc0e102042542392ca17c1 Mon Sep 17 00:00:00 2001 From: "Thomas Neep (Advanced Research Computing)" Date: Tue, 3 Mar 2026 16:01:08 +0000 Subject: [PATCH 6/7] Change POSTGRES environment variables --- agate/core/production.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/agate/core/production.py b/agate/core/production.py index f09c8bb..f5090c9 100644 --- a/agate/core/production.py +++ b/agate/core/production.py @@ -188,10 +188,10 @@ DATABASES = { "default": { "ENGINE": "django.db.backends.postgresql", - "NAME": os.environ["DATABASE_NAME"], - "USER": os.environ["DATABASE_USER"], - "PASSWORD": os.environ["DATABASE_PASSWORD"], - "HOST": os.environ["DATABASE_HOST"], + "NAME": os.environ["POSTGRES_DB"], + "USER": os.environ["POSTGRES_USER"], + "PASSWORD": os.environ["POSTGRES_PASSWORD"], + "HOST": os.environ["POSTGRES_HOST"], } } From 1eb6d1fe4f95ad4d57d1574c314a869b58ce3a6c Mon Sep 17 00:00:00 2001 From: "Thomas Neep (Advanced Research Computing)" Date: Tue, 10 Mar 2026 09:55:28 +0000 Subject: [PATCH 7/7] Fix bugs in settings --- agate/.dockerignore | 2 +- agate/core/production.py | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/agate/.dockerignore b/agate/.dockerignore index 1951104..36b1bd3 100644 --- a/agate/.dockerignore +++ b/agate/.dockerignore @@ -39,5 +39,5 @@ coverage_html_report/ *log # The varys_config.cfg needs to be given when setting up the container -e.g. with a volume mount or some other technique +# e.g. with a volume mount or some other technique varys_config.cfg diff --git a/agate/core/production.py b/agate/core/production.py index f5090c9..8431867 100644 --- a/agate/core/production.py +++ b/agate/core/production.py @@ -110,7 +110,7 @@ DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' -DEBUG = os.environ.get("DEBUG", False) +DEBUG = os.getenv("DEBUG", False) # Storages @@ -195,8 +195,8 @@ } } -CORS_ALLOWED_ORIGINS = os.environ.get("CORS_ALLOWED_ORIGINS", "").split(",") -CSRF_TRUSTED_ORIGINS = os.environ.get("CSRF_TRUSTED_ORIGINS", "").split(",") +CORS_ALLOWED_ORIGINS = [i for i in os.getenv("CORS_ALLOWED_ORIGINS", "").split(",") if i] +CSRF_TRUSTED_ORIGINS = [i for i in os.getenv("CSRF_TRUSTED_ORIGINS", "").split(",") if i] ONYX_DOMAIN = os.environ["ONYX_DOMAIN"] @@ -210,6 +210,4 @@ auto_acknowledge=False ) -LIMITED_PROJECT_LIST = os.environ.get( - "LIMITED_PROJECT_LIST", "synthscape", -).split(",") +LIMITED_PROJECT_LIST = os.getenv("LIMITED_PROJECT_LIST", "synthscape").split(",")