PB-2158: implement otel instrumentation

.env.ci

@@ -5,4 +5,7 @@ BOD_DB_NAME=bod_local
 BOD_DB_HOST=db
 BOD_DB_PORT=5432
 BOD_DB_USER=local-db-user
-BOD_DB_PASSWD=local-db-user
+BOD_DB_PASSWD=local-db-user
+
+# OTEL
+OTEL_SDK_DISABLED=true

.env.local

@@ -15,4 +15,15 @@ BOD_DB_PASSWD=local-db-user
 
 SQLALCHEMY_TRACK_MODIFICATIONS=True
 SQLALCHEMY_POOL_PRE_PING=True
-SQLALCHEMY_ISOLTATION_LEVEL="AUTOCOMMIT"
+SQLALCHEMY_ISOLTATION_LEVEL="AUTOCOMMIT"
+
+# OTEL
+OTEL_SDK_DISABLED=false
+OTEL_ENABLE_BOTOCORE=true
+OTEL_ENABLE_FLASK=true
+OTEL_ENABLE_LOGGING=true
+OTEL_ENABLE_REQUESTS=true
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
+OTEL_EXPORTER_OTLP_INSECURE=true
+OTEL_RESOURCE_ATTRIBUTES=service.name=service-wmts
+OTEL_PYTHON_EXCLUDED_URLS="checker"

.pylintrc

@@ -36,10 +36,6 @@ load-plugins=pylint_sqlalchemy,pylint_flask
 # Pickle collected data for later comparisons.
 persistent=yes
 
-# When enabled, pylint would attempt to guess common misconfiguration and emit
-# user-friendly hints instead of false-positive error messages.
-suggestion-mode=yes
-
 # Allow loading of arbitrary C extensions. Extensions are imported into the
 # active Python interpreter and may run arbitrary code.
 unsafe-load-any-extension=no
@@ -197,7 +193,7 @@ indent-after-paren=4
 indent-string='    '
 
 # Maximum number of characters on a single line.
-max-line-length=80
+max-line-length=100
 
 # Maximum number of lines in a module.
 max-module-lines=1000

Makefile

@@ -60,6 +60,7 @@ help:
 	@echo -e " \033[1mSetup TARGETS\033[0m "
 	@echo "- setup              Create the python virtual environment with developper tools and activate it"
 	@echo "- ci                 Create the python virtual environment and install requirements based on the Pipfile.lock"
+	@echo "- otelrequirements   Get a list of available otel instrumentation libraries to add to the pipfile of this project"
 	@echo -e " \033[1mFORMATING, LINTING AND TESTING TOOLS TARGETS\033[0m "
 	@echo "- format             Format the python source code"
 	@echo "- ci-check-format    Format the python source code and check if any files has changed. This is meant to be used by the CI."
@@ -95,6 +96,10 @@ ci: $(REQUIREMENTS)
 	# Create virtual env with all packages for development using the Pipfile.lock
 	pipenv sync --dev
 
+.PHONY: otelrequirements
+otelrequirements:
+	edot-bootstrap --action=requirements
+
 # linting target, calls upon yapf to make sure your code is easier to read and respects some conventions.
 
 .PHONY: format
@@ -137,7 +142,7 @@ serve: clean_logs $(LOGS_DIR)
 
 .PHONY: gunicornserve
 gunicornserve: clean_logs $(LOGS_DIR)
-	LOGS_DIR=$(LOGS_DIR) $(PYTHON) wsgi.py
+	OTEL_SDK_DISABLED=true LOGS_DIR=$(LOGS_DIR) $(PYTHON) wsgi.py
 
 
 # Docker related functions.

Pipfile

@@ -8,7 +8,7 @@ PyYAML = "~=6.0"
 gevent = "~=25.5"
 gunicorn = "~=23.0"
 Flask = "~=3.1"
-logging-utilities = "~=5.0"
+logging-utilities = "~=5.3.0"
 boto3 = "~=1.40"
 pyproj = "~=3.7"
 requests = "~=2.32"
@@ -18,6 +18,13 @@ psycopg = {extras = ["binary"], version = "~=3.2"}
 SQLAlchemy = "~=2.0"
 Flask-SQLAlchemy = "~=3.1"
 gatilegrid = "~=1.0"
+# OpenTelemetry packages
+opentelemetry-sdk = "*"
+opentelemetry-exporter-otlp = "*"
+opentelemetry-instrumentation-botocore = "*"
+opentelemetry-instrumentation-flask = "*"
+opentelemetry-instrumentation-logging = "*"
+opentelemetry-instrumentation-requests = "*"
 
 [dev-packages]
 yapf = "*"

app/helpers/otel.py

@@ -0,0 +1,51 @@
+from os import getenv
+
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import \
+    OTLPSpanExporter
+from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
+from opentelemetry.instrumentation.flask import FlaskInstrumentor
+from opentelemetry.instrumentation.logging import LoggingInstrumentor
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+from app.helpers.utils import strtobool
+
+
+def initialize() -> None:
+    if not strtobool(getenv("OTEL_SDK_DISABLED", "false")):
+        if strtobool(getenv("OTEL_ENABLE_BOTOCORE", "false")):
+            BotocoreInstrumentor().instrument()
+        if strtobool(getenv("OTEL_ENABLE_LOGGING", "false")):
+            LoggingInstrumentor().instrument()
+        if strtobool(getenv("OTEL_ENABLE_REQUESTS", "false")):
+            RequestsInstrumentor().instrument()
+
+
+def initialize_flask(app):
+    if not strtobool(getenv("OTEL_SDK_DISABLED", "false")):
+        if strtobool(getenv("OTEL_ENABLE_FLASK", "false")):
+            FlaskInstrumentor().instrument_app(app)
+
+
+def setup_trace_provider():
+    if not strtobool(getenv("OTEL_SDK_DISABLED", "false")):
+        # Since we created a new tracer, the default span processor is gone. We need to
+        # create a new one using the default OTEL env variables and ad it to the tracer.
+        span_processor = BatchSpanProcessor(
+            OTLPSpanExporter(
+                endpoint=getenv(
+                    'OTEL_EXPORTER_OTLP_ENDPOINT', "http://localhost:4317"
+                ),
+                headers=getenv('OTEL_EXPORTER_OTLP_HEADERS'),
+                insecure=strtobool(
+                    getenv('OTEL_EXPORTER_OTLP_INSECURE', "false")
+                )
+            )
+        )
+
+        provider = TracerProvider(resource=Resource.create())
+        provider.add_span_processor(span_processor)
+        trace.set_tracer_provider(provider)

app/helpers/utils.py

@@ -104,3 +104,18 @@ def get_default_tile_matrix_set(epsg):
     tilematrix_set['MAXY'] = gagrid.MAXY if int(epsg) == 4326 else gagrid.MINX
     tilematrix_set['MINX'] = gagrid.MINX if int(epsg) == 4326 else gagrid.MAXY
     return tilematrix_set
+
+
+def strtobool(val):
+    """Convert a string representation of truth to True or False.
+
+    True values are 'y', 'yes', 't', 'true', 'on', and '1';
+    False values are 'n', 'no', 'f', 'false', 'off', and '0'.
+    Raises ValueError if 'val' is anything else.
+    """
+    val = val.lower()
+    if val in ('y', 'yes', 't', 'true', 'on', '1'):
+        return True
+    if val in ('n', 'no', 'f', 'false', 'off', '0'):
+        return False
+    raise ValueError(f"invalid truth value: {val}")

config/otel-local-config.yaml

@@ -0,0 +1,21 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+
+processors:
+  batch:
+
+exporters:
+  debug:
+    verbosity: detailed
+  zipkin:
+    endpoint: http://zipkin:9411/api/v2/spans
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [debug, zipkin]

docker-compose-otel.yml

@@ -0,0 +1,14 @@
+version: "2"
+services:
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    command: --config otel-local-config.yaml
+    volumes:
+      - ./config/otel-local-config.yaml:/otel-local-config.yaml
+    ports:
+      - "4317:4317"
+
+  zipkin:
+    image: openzipkin/zipkin:latest
+    ports:
+      - "9411:9411"

wsgi.py

@@ -7,11 +7,23 @@
 import, especially the app import, which would cause the boto module to be
 loaded, which would in turn load the ssl module.
 """
-# pylint: disable=wrong-import-position,wrong-import-order
+# pylint: disable=wrong-import-position,wrong-import-order,ungrouped-imports
 import gevent.monkey
 
 gevent.monkey.patch_all()
 
+# Initialize OTEL.
+# Initialize should be called as early as possible, but at least before the app is imported
+# The order has a impact on how the libraries are instrumented. If called after app import,
+# e.g. the flask instrumentation has no effect. See:
+# https://github.com/open-telemetry/opentelemetry.io/blob/main/content/en/docs/zero-code/python/troubleshooting.md#use-programmatic-auto-instrumentation
+
+from app.helpers.otel import initialize
+from app.helpers.otel import initialize_flask
+from app.helpers.otel import setup_trace_provider
+
+initialize()
+
 from gunicorn.app.base import BaseApplication
 
 from app.app import app as application
@@ -25,6 +37,15 @@
 from app.settings import WMTS_WORKERS
 from app.settings import WSGI_TIMEOUT
 
+initialize_flask(application)
+
+
+def post_fork(server, worker):
+    server.log.info("Worker spawned (pid: %s)", worker.pid)
+
+    # Setup OTEL providers for this worker
+    setup_trace_provider()
+
 
 class StandaloneApplication(BaseApplication):
     # pylint: disable=abstract-method
@@ -65,6 +86,7 @@ def on_starting(server):
         'workers': WMTS_WORKERS,
         'worker_tmp_dir': GUNICORN_WORKER_TMP_DIR,
         'timeout': WSGI_TIMEOUT,
+        'post_fork': post_fork,
         'keepalive': GUNICORN_KEEPALIVE,
         'access_log_format':
             '%(h)s %(l)s %(u)s "%(r)s" %(s)s %(B)s Bytes '