diff --git a/.circleci/config.yml b/.circleci/config.yml index e4aac72a0..a0a9840ad 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -124,6 +124,10 @@ jobs: - checkout - attach_workspace: at: *working_directory + - restore_cache: + keys: + - temboard-{{ .Branch }} + - temboard-master - run: name: Build Agent & UI command: | @@ -139,6 +143,10 @@ jobs: - agent/dist/*.whl - ui/dist/*.tar.gz - ui/dist/*.whl + - ui/build/bin/prometheus + - save_cache: + key: temboard-{{ .Branch }} + paths: ["dev/downloads/"] lint-python: docker: [image: python:3.9] @@ -265,8 +273,8 @@ jobs: - persist_to_workspace: root: . paths: - - agent/dist/*.noarch.rpm - - ui/dist/*.noarch.rpm + - agent/dist/temboard-agent-*.noarch.rpm + - ui/dist/temboard-*.rpm deb: parameters: diff --git a/.config/temboard.conf b/.config/temboard.conf index 74de0c6c8..56dee556a 100644 --- a/.config/temboard.conf +++ b/.config/temboard.conf @@ -49,7 +49,7 @@ method = stderr level = DEBUG [monitoring] -prometheus = dev/bin/prometheus +prometheus = ui/build/bin/prometheus [notifications] # SMTP host diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 17ace3efa..81756998c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -471,30 +471,6 @@ and if not found, tries to download wheel from PyPI. Use top level `make dist` to generate wheels. -## Testing with Grafana - -temBoard exposes /metrics in OpenMetric format. Development environment -includes Prometheus and Grafana preconfigured with authenticated access and -file-based service discovery. You only need a couple of steps to view temBoard -metrics in Grafana. - -- Start development temBoard UI and at lease one of agent0 or agent1. -- Generate Prometheus targets using `make prom-targets`. -- Go to [development Grafana import page](http://localhost:3000/dashboard/import). -- Import Dashboard ID [1860](https://grafana.com/grafana/dashboards/1860) - (*Node Exporter Full*) or [9628](https://grafana.com/grafana/dashboards/1860) - (*PostgreSQL Database*). -- Now wait a few minutes that Prometheus scrapes data. Scrape interval is 1 - minute. - -temBoard exposes only a subset of metrics from node\_exporter and -postgres\_exporter, plus a set of custom metrics prefixed with `x`, e.g. -`xpg_wal_files_bytes_total`. Thus a lot of panel will show *N/A*. - -Prometheus also watch `dev/prometheus/targets/custom.yaml` files for hand -written targets. - - ## Investigate logs with lnav [lnav] is an awesome tool to browse and analyze log files. diff --git a/Makefile b/Makefile index e4e6212c2..1b0f99bda 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ develop-%:: .env @dev/bin/checkdocker $(DOCKER_MAX_VERSION) git config blame.ignoreRevsFile .git-blame-ignore-revs if [ -d ~/.config/lnav/formats ] ; then ln -fsTv $$PWD/dev/lnav/formats ~/.config/lnav/formats/temboard ; fi - $(MAKE) -j 2 install-$* dev/bin/prometheus + $(MAKE) -j 2 install-$* ui/build/bin/prometheus ui/build/bin/promtool mkdir -p dev/temboard cd ui/; npm install-clean cd ui/; npm run build @@ -56,20 +56,21 @@ install-%: venv-% dev/venv-py$*/bin/temboard-agent --version # smoke test # LTS -PROMETHEUS_VERSION=2.45.1 +PROMETHEUS_VERSION=2.53.0 dev/downloads/prometheus-%.linux-amd64.tar.gz: mkdir -p $(dir $@) curl --fail --silent -L "https://github.com/prometheus/prometheus/releases/download/v$*/$(notdir $@)" --output $@ -dev/bin/prometheus dev/bin/promtool: dev/downloads/prometheus-$(PROMETHEUS_VERSION).linux-amd64.tar.gz +ui/build/bin/prometheus ui/build/bin/promtool: dev/downloads/prometheus-$(PROMETHEUS_VERSION).linux-amd64.tar.gz + mkdir -p $(dir $@) tar --extract --file "$<" --directory "$(dir $@)" --strip-component=1 --touch "prometheus-$(PROMETHEUS_VERSION).linux-amd64/$(notdir $@)" "$@" --version # Smoketest clean: #: Trash venv and containers. docker compose down --volumes --remove-orphans docker rmi --force dalibo/temboard-agent:dev - rm -rf dev/venv-py* .venv-py* dev/build/ dev/prometheus/targets/temboard-dev.yaml - rm -vf dev/bin/prometheus dev/bin/promtool + rm -rf dev/venv-py* .venv-py* dev/build/ + rm -vf ui/build/bin/prometheus ui/build/bin/promtool rm -rf agent/build/ .env agent/.coverage rm -rvf ui/build/ ui/.coverage $(MAKE) clean-static @@ -127,10 +128,6 @@ tests: #: Execute all tests. clean-tests: #: Clean tests runtime files rm -rf tests/downloads/ tests/logs/ tests/screenshots/ -prom-targets: dev/prometheus/targets/temboard-dev.yaml #: Generate Prometheus dev targets. -dev/prometheus/targets/temboard-dev.yaml: dev/bin/mktargets .env - $^ > $@ - VERSION=$(shell cd ui; python3 setup.py --version) BRANCH?=master # When stable branch v8 is created, use this: @@ -162,7 +159,7 @@ release: #: Tag and push a new git release. release-notes: #: Extract changes for current release FINAL_VERSION="$(shell echo $(VERSION) | grep -Po '([^a-z]{3,})')" ; sed -En "/Unreleased/d;/^#+ $$FINAL_VERSION/,/^#/p" CHANGELOG.md | sed '1d;$$d' -dist: #: Build sources and wheels. +dist: ui/build/bin/prometheus #: Build sources and wheels. cd agent/; python3 setup.py sdist bdist_wheel test -f ui/temboardui/static/dist/.vite/manifest.json cd ui/; python3 setup.py sdist bdist_wheel --universal diff --git a/agent/temboardagent/web/service.py b/agent/temboardagent/web/service.py index b0ccbf9ef..ab69c458a 100644 --- a/agent/temboardagent/web/service.py +++ b/agent/temboardagent/web/service.py @@ -22,7 +22,7 @@ def __str__(self): return self.name # for services.run - def setup(self): + def setup(self, *_, **__): ServerHandler.server_software = "temBoard-agent/%s" % __version__ try: diff --git a/dev/bin/mktargets b/dev/bin/mktargets deleted file mode 100755 index fa7b0908e..000000000 --- a/dev/bin/mktargets +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -eu - -# shellcheck source=/dev/null -. .env - -target="${TEMBOARD_UI_URL#https://}" - -cat <<-EOF -- targets: - - $target - labels: - __metrics_path__: /proxy/0.0.0.0/2345/monitoring/metrics - instance: "postgres0.dev:5432" - # For PostgreSQL Database dashboard #9628 - kubernetes_namespace: nok8s - release: norelease -- targets: - - $target - labels: - __metrics_path__: /proxy/0.0.0.0/2346/monitoring/metrics - instance: "postgres1.dev:5432" - kubernetes_namespace: nok8s - release: norelease -EOF diff --git a/dev/grafana/rootfs/etc/grafana/provisioning/dashboards/perf.yaml b/dev/grafana/rootfs/etc/grafana/provisioning/dashboards/perf.yaml index 7bf753ee0..3abb04797 100644 --- a/dev/grafana/rootfs/etc/grafana/provisioning/dashboards/perf.yaml +++ b/dev/grafana/rootfs/etc/grafana/provisioning/dashboards/perf.yaml @@ -7,15 +7,3 @@ providers: allowUiUpdates: true options: path: /usr/share/temboard/perf.json -- name: "Postgres from temBoard agent" - type: file - folder: '' - allowUiUpdates: true - options: - path: /usr/share/temboard/postgres.json -- name: "temBoard Prometheus" - type: file - folder: '' - allowUiUpdates: true - options: - path: /usr/share/temboard/temboard-prometheus.json diff --git a/dev/grafana/rootfs/usr/share/temboard/postgres.json b/dev/grafana/rootfs/usr/share/temboard/postgres.json deleted file mode 100644 index 3bbabc85e..000000000 --- a/dev/grafana/rootfs/usr/share/temboard/postgres.json +++ /dev/null @@ -1,201 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 2, - "iteration": 1643970369705, - "links": [], - "liveNow": false, - "panels": [ - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 50, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "01lWBBhnk" - }, - "exemplar": true, - "expr": "sum(rate(xacts_n_commit{logfile=\"$logfile\"}[5m]))", - "interval": "", - "legendFormat": "commit", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "01lWBBhnk" - }, - "exemplar": true, - "expr": "sum(rate(xacts_n_rollback{logfile=\"$logfile\"}[5m]))", - "hide": false, - "interval": "", - "legendFormat": "rollback", - "refId": "B" - } - ], - "title": "TPS", - "type": "timeseries" - }, - { - "datasource": { - "type": "loki", - "uid": "j1tzJKb7k" - }, - "gridPos": { - "h": 13, - "w": 24, - "x": 0, - "y": 7 - }, - "id": 2, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": false, - "sortOrder": "Ascending", - "wrapLogMessage": false - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "j1tzJKb7k" - }, - "expr": "{logfile=~\"$logfile\"}", - "refId": "A" - } - ], - "title": "Messages", - "type": "logs" - } - ], - "refresh": false, - "schemaVersion": 33, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": ".+", - "value": ".+" - }, - "definition": "label_values(logfile)", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "logfile", - "options": [], - "query": { - "query": "label_values(logfile)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - } - ] - }, - "time": { - "from": "now-3h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Postgres", - "uid": "gQ7FEi-7z", - "version": 3, - "weekStart": "" -} diff --git a/dev/grafana/rootfs/usr/share/temboard/temboard-prometheus.json b/dev/grafana/rootfs/usr/share/temboard/temboard-prometheus.json deleted file mode 100644 index 7ad8c801d..000000000 --- a/dev/grafana/rootfs/usr/share/temboard/temboard-prometheus.json +++ /dev/null @@ -1,137 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 3, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "Kme6NfMSz" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Kme6NfMSz" - }, - "editorMode": "builder", - "expr": "node_load1", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Panel Title", - "type": "timeseries" - } - ], - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "temBoard Prometheus", - "uid": "aC9VOfMSk", - "version": 1, - "weekStart": "" -} diff --git a/docker-compose.yml b/docker-compose.yml index febc6d80b..0f629ec7f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -214,7 +214,6 @@ services: volumes: - ./dev/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - ./dev/prometheus/import:/import - - ./dev/prometheus/targets:/targets ports: - 9090:9090 command: diff --git a/tests/test_00_setup_ui.py b/tests/test_00_setup_ui.py index 0d05027e1..d436eed11 100644 --- a/tests/test_00_setup_ui.py +++ b/tests/test_00_setup_ui.py @@ -119,7 +119,7 @@ def test_proctitle(ui): continue assert cmdline.startswith("temboard: "), cmdline - assert ": worker" in cmdline or ": scheduler" in cmdline + assert ": web" not in cmdline def test_autossl(ui): diff --git a/tests/test_10_setup_agent.py b/tests/test_10_setup_agent.py index 7f0b70055..c580d980a 100644 --- a/tests/test_10_setup_agent.py +++ b/tests/test_10_setup_agent.py @@ -68,8 +68,7 @@ def test_proctitle(agent): cmdline = fo.read() assert cmdline.startswith("temboard-agent: temboard-tests: ") - - assert ": worker" in cmdline or ": scheduler" in cmdline + assert ": web" not in cmdline def test_discover_http(agent, agent_env, pg_version): diff --git a/ui/packaging/deb/mkdeb.sh b/ui/packaging/deb/mkdeb.sh index dcd1061ac..296b1968c 100755 --- a/ui/packaging/deb/mkdeb.sh +++ b/ui/packaging/deb/mkdeb.sh @@ -61,6 +61,7 @@ mv "$DESTDIR/usr/local/lib"/*/dist-packages "$DESTDIR/usr/lib/temboard" mv "$DESTDIR/usr/local/lib/systemd" "$DESTDIR/usr/lib" # Move binaries out of FHS. mv "$DESTDIR/usr/local/bin" "$DESTDIR/usr/lib/temboard/" +cp build/bin/prometheus "$DESTDIR/usr/lib/temboard/" rm -rf "${DESTDIR:?}/usr/local" # Create FHS wrapper. @@ -101,6 +102,7 @@ apt-get install --yes --no-install-recommends "./$deb" cd / temboard --version test -f /usr/lib/temboard/temboardui/static/dist/.vite/manifest.json + test -x /usr/lib/temboard/prometheus test -x /usr/share/temboard/auto_configure.sh test -f /usr/lib/systemd/system/temboard.service ) diff --git a/ui/packaging/rpm/build.sh b/ui/packaging/rpm/build.sh index 846909e38..fe342c507 100755 --- a/ui/packaging/rpm/build.sh +++ b/ui/packaging/rpm/build.sh @@ -10,4 +10,8 @@ test -f "dist/temboard-$VERSION.tar.gz" cp --preserve --force packaging/rpm/temboard.spec /tmp/temboard.spec sed -i "/^Version:/s/GENERATED/$VERSION/" /tmp/temboard.spec export BUILDDIR="$PWD/dist/" SMOKETEST=1 -exec rpmbuild.sh /tmp/temboard.spec "dist/temboard-$VERSION.tar.gz" +# Globally disable debug package generation. +cat >>/etc/rpm/macros.temboard < - 8.0-1 diff --git a/ui/share/auto_configure.sh b/ui/share/auto_configure.sh index e0c89fa07..a4540d4ae 100755 --- a/ui/share/auto_configure.sh +++ b/ui/share/auto_configure.sh @@ -126,12 +126,33 @@ generate_configuration() { [monitoring] # purge_after = 730 + prometheus = $(find_prometheus) [statements] # purge_after = 7 EOF } +find_prometheus() { + local bin + bin=/usr/lib/temboard/prometheus + if [ -x "$bin" ] ; then + echo "$bin" + return + fi + if bin="$(type -p prometheus)" ; then + echo "$bin" + return + fi + # .../temboard/workdir/etc/ui/... + bin="$ETCDIR/../../../ui/build/prometheus" + if [ -x "$bin" ] ; then + echo "$bin" + return + fi + return 1 +} + pwgen() { # Generates a random password of 32 hexadecimal characters. od -vN $((${1-32} / 2)) -An -tx1 /dev/urandom | tr -d ' \n' diff --git a/ui/temboardui/__main__.py b/ui/temboardui/__main__.py index 3d3987f92..644f46a1f 100644 --- a/ui/temboardui/__main__.py +++ b/ui/temboardui/__main__.py @@ -34,9 +34,8 @@ def main(*a, **kw): # Load sub commands. __import__(__package__ + ".cli.apikey") __import__(__package__ + ".cli.generate_key") - # Don't import prometheus command yet. - # Use python -m temboardui.cli.prometheus until 9.0. __import__(__package__ + ".cli.migratedb") + __import__(__package__ + ".cli.prometheus") __import__(__package__ + ".cli.query_agent") __import__(__package__ + ".cli.register_instance") __import__(__package__ + ".cli.routes") diff --git a/ui/temboardui/cli/apikey.py b/ui/temboardui/cli/apikey.py index e6e2f56c8..8c8120f51 100644 --- a/ui/temboardui/cli/apikey.py +++ b/ui/temboardui/cli/apikey.py @@ -1,7 +1,5 @@ import logging -import string import sys -from secrets import choice from textwrap import dedent from ..model import Session @@ -31,7 +29,7 @@ def define_arguments(self, parser): def main(self, args): session = Session() key = ( - ApiKeys.insert(secret=generate_secret(), comment=args.comment) + ApiKeys.insert(secret=ApiKeys.generate_secret(), comment=args.comment) .with_session(session) .scalar() ) @@ -101,10 +99,3 @@ def main(self, args): logger.info("Purged %d keys.", count) else: logger.info("No expired keys to purge.") - - -_SECRET_LETTERS = string.ascii_letters + string.digits + "+/-_" - - -def generate_secret(length=40): - return "".join(choice(_SECRET_LETTERS) for _ in range(length)) diff --git a/ui/temboardui/cli/app.py b/ui/temboardui/cli/app.py index 0b7e4ac84..bbc0ee51b 100644 --- a/ui/temboardui/cli/app.py +++ b/ui/temboardui/cli/app.py @@ -1,5 +1,6 @@ import logging.config import os +import shutil import sys from argparse import _VersionAction from concurrent.futures import ThreadPoolExecutor @@ -12,6 +13,7 @@ from tornado.httpserver import HTTPServer from tornado.wsgi import WSGIContainer +from .. import prometheus from ..autossl import AutoHTTPSServer from ..core import workers from ..model import QUERIES @@ -106,6 +108,10 @@ def main(self, argv, environ): self.webservice = TornadoService(self) + self.prometheus = prometheus.Manager(app=self) + # Not appending prometheus to services, because prometheus does not + # need apply_config. + # TaskList engine setup must be done before we load the plugins self.scheduler.task_list_engine = TaskListSQLite3Engine( os.path.join(self.config.temboard["home"], "server_tasks.db") @@ -132,7 +138,7 @@ def apply_config(self): self.tornado_app.engine = configure_db_session(self.config.repository) def log_versions(self): - versions = inspect_versions() + versions = inspect_versions(prometheusbin=self.config.monitoring.prometheus) logger.debug("Running on %s %s.", versions["distname"], versions["distversion"]) logger.debug( "Using Python %s (%s) and Tornado %s .", @@ -146,6 +152,7 @@ def log_versions(self): versions["psycopg2"], versions["sqlalchemy"], ) + logger.debug("Using Prometheus %s.", versions["prometheus"]) class TemboardUIConfiguration(MergedConfiguration): @@ -270,7 +277,6 @@ def __init__(self, app): self.app = app # Ref to services.BackgroundManager self.background = None - # For services.run() self.perf = perf.PerfCounters.setup(service=self.name) def __str__(self): @@ -280,8 +286,10 @@ def __str__(self): def create_loop(self): return tornado.ioloop.IOLoop.instance() - def setup(self): + def setup(self, sgm, bg): + self.background = bg if self.perf: + sgm.register(self.perf) self.perf.run() flask_app.vitejs.read_manifest() @@ -347,9 +355,10 @@ def _setup_autoreload(self): autoreload.watch(path) def _autoreload_hook(self): - if self.background: - logger.debug("Stopping background service before reloading.") - self.background.stop() + if not self.background: + return + logger.debug("Stopping background service before reloading.") + self.background.stop() def _iter_template_files(self): rootpkg = __import__(__name__) @@ -436,6 +445,8 @@ def list_options_specs(): s = "monitoring" yield OptionSpec(s, "purge_after", default=730, validator=v.nday) + prometheus = shutil.which("prometheus") or OptionSpec.REQUIRED + yield OptionSpec(s, "prometheus", default=prometheus, validator=v.file_) s = "statements" yield OptionSpec(s, "purge_after", default=7, validator=v.nday) diff --git a/ui/temboardui/cli/prometheus.py b/ui/temboardui/cli/prometheus.py index d58225eb8..c5172bfa4 100644 --- a/ui/temboardui/cli/prometheus.py +++ b/ui/temboardui/cli/prometheus.py @@ -12,20 +12,12 @@ # `temBoard prometheus` with a dedicated dashboard. # import logging -import os -import signal -import subprocess import sys -from datetime import datetime -import jinja2 - -import temboardui - -from ..model import Session, orm +from .. import prometheus +from ..toolkit import services from ..toolkit.app import SubCommand from ..toolkit.errors import UserError -from ..toolkit.services import Service from .app import app logger = logging.getLogger(__package__ + ".prometheus") @@ -39,167 +31,13 @@ class Prometheus(SubCommand): """ + is_service = True + def main(self, args): if not self.app.config.monitoring.prometheus: raise UserError("missing prometheus binary") - prometheus = PrometheusService(app=self.app) - with prometheus: - prometheus.run() - - -class PrometheusService(Service): - def __init__(self, **kwargs): - super().__init__(name="prometheus manager", **kwargs) - self.proc = None - - @property - def pidfile(self): - return self.app.config.temboard.home + "/prometheus.pid" - - @property - def home(self): - return self.app.config.temboard.home + "/prometheus" - - def serve(self): - self.terminate_running_prometheus(self.pidfile) - - logger.info("Provisionning Prometheus in %s.", self.home) - provision_prometheus(self.app, self.home) - logger.info("Starting %s.", self.app.config.monitoring.prometheus) - cmd = [ - self.app.config.monitoring.prometheus, - f"--config.file={self.home}/prometheus.yml", - # 0.0.0.0 is reachable from dev grafana for testing purpose. - # Once we have alertmanager, use localhost. - "--web.listen-address=0.0.0.0:8890", - "--log.level=debug", - "--storage.tsdb.retention.time=1h", - ] - self.proc = subprocess.Popen(cmd, cwd=self.home) - logger.info("Started Prometheus as PID %s.", self.proc.pid) - with open(self.pidfile, "w") as fo: - fo.write(str(self.proc.pid)) - self.setup_instances() - logger.debug("Waiting prometheus service forever.") - self.proc.wait() - - def terminate_running_prometheus(self, pidfile): - if not os.path.exists(pidfile): - return - - with open(pidfile) as fo: - pid = fo.read() - - try: - logger.debug("Eventually terminating Prometheus PID %s.", pid) - os.kill(int(pid), signal.SIGTERM) - logger.info("Prometheus running as PID %s. Terminating.", pid) - except OSError: - logger.debug("Spurious PID file %s.", pidfile) - - def setup_instances(self): - session = Session() - for (instance,) in session.execute(orm.Instances.all()): - if not instance.discover: - logger.debug("Skipping unreachable instance %s.", instance) - continue - if instance.discover["temboard"]["agent_version"] < "8": - logger.debug("Skipping old agent %s.", instance) - continue - provision_instance(self.app, instance) - self.proc.send_signal(signal.SIGHUP) - - def reload(self): - super().reload() - if self.proc: - self.proc.send_signal(signal.SIGHUP) - - def teardown(self): - if not self.proc: - return - logger.info("Stopping Prometheus.") - self.proc.terminate() - self.proc = None - os.unlink(self.app.config.temboard.home + "/prometheus.pid") - - -def provision_prometheus(app, home): - logger.debug("Ensure prometheus home is created.") - os.makedirs(home + "/instances.d", exist_ok=True) - t = jinja2.Template( - source=PROMETHEUS_CONFIG_TEMPLATE, - trim_blocks=True, - lstrip_blocks=True, - undefined=jinja2.StrictUndefined, - keep_trailing_newline=True, - ) - conf = t.render( - app=app, - apikey="UNSECURE_DEV_APIKEY", - home=home, - now=datetime.utcnow(), - temboard_version=temboardui.__version__, - ) - with open(home + "/prometheus.yml", "w") as fo: - os.chmod(fo.name, 0o600) - fo.write(conf) - - -PROMETHEUS_CONFIG_TEMPLATE = """\ -# This file is generated by temBoard. Do not edit. -# -# Generation Date: {{ now }} -# temBoard Version: {{ temboard_version }} -# -scrape_configs: -- job_name: temboard - scrape_interval: 60s - scheme: http - authorization: - type: Bearer - credentials: {{ apikey }} - file_sd_configs: - - files: - - instances.d/instance-*.yml -""" - - -def provision_instance(app, instance): - logger.info("Provisionning %s.", instance) - t = jinja2.Template( - source=INSTANCE_CONFIG_TEMPLATE, - trim_blocks=True, - lstrip_blocks=True, - undefined=jinja2.StrictUndefined, - keep_trailing_newline=True, - ) - conf = t.render( - app=app, - instance=instance, - now=datetime.utcnow(), - temboard_version=temboardui.__version__, - ) - path = "{}/prometheus/instances.d/instance-{}-{}.scrape.yml".format( - app.config.temboard.home, instance.agent_address, instance.agent_port - ) - with open(path, "w") as fo: - os.chmod(fo.name, 0o600) - fo.write(conf) - - -INSTANCE_CONFIG_TEMPLATE = """\ -# This file is generated by temBoard. Do not edit. -# -# Generation Date: {{ now }} -# temBoard Version: {{ temboard_version }} -# -- targets: [localhost:8888] - labels: - __metrics_path__: "/proxy/{{ instance.agent_address }}/{{ instance.agent_port }}/monitoring/metrics" - instance: "{{ instance.hostname }}:{{ instance.pg_port }}" - agent: "{{ instance.agent_address }}:{{ instance.agent_port }}" -""" # noqa + return services.run(prometheus.Manager(app=self.app)) if "__main__" == __name__: diff --git a/ui/temboardui/cli/serve.py b/ui/temboardui/cli/serve.py index e1065c11a..5d3b2913b 100644 --- a/ui/temboardui/cli/serve.py +++ b/ui/temboardui/cli/serve.py @@ -14,4 +14,9 @@ def main(self, args): check_schema() self.app.config.load_signing_key() - services.run(self.app.webservice, self.app.scheduler, self.app.worker_pool) + services.run( + self.app.webservice, + self.app.scheduler, + self.app.worker_pool, + self.app.prometheus, + ) diff --git a/ui/temboardui/handlers/home.py b/ui/temboardui/handlers/home.py index 16017dda7..618aa77d5 100644 --- a/ui/temboardui/handlers/home.py +++ b/ui/temboardui/handlers/home.py @@ -16,7 +16,7 @@ def home(request): @app.route("/about") def about(request): - versions_info = inspect_versions() + versions_info = inspect_versions(prometheusbin=request.config.monitoring.prometheus) instances = request.db_session.scalar(Instances.count()) roles = request.db_session.scalar(Roles.count()) infos = { @@ -29,6 +29,7 @@ def about(request): "Tornado": versions_info["tornado"], "libpq": versions_info["libpq"], "psycopg2": versions_info["psycopg2"], + "Prometheus": "%(prometheus)s (%(prometheusbin)s)" % versions_info, "SQLAlchemy": versions_info["sqlalchemy"], "Instances": instances, "Users": roles, diff --git a/ui/temboardui/handlers/settings/instance.py b/ui/temboardui/handlers/settings/instance.py index 0702fbeb2..a80e718d0 100644 --- a/ui/temboardui/handlers/settings/instance.py +++ b/ui/temboardui/handlers/settings/instance.py @@ -57,6 +57,7 @@ def create_instance_helper(request, data): if "monitoring" in plugins: from ...plugins.monitoring import collector as monitoring_collector + app.prometheus.trigger_reload() logger.info("Schedule monitoring collect for agent now.") monitoring_collector.defer( app, address=data["new_agent_address"], port=data["new_agent_port"] @@ -180,6 +181,7 @@ def json_delete_instance(request): if not data.get("agent_port"): raise HTTPError(400, "Agent port field is missing.") delete_instance(request.db_session, **data) + request.handler.application.temboard_app.prometheus.trigger_reload() return {"delete": True} diff --git a/ui/temboardui/model/orm.py b/ui/temboardui/model/orm.py index 2ee71161c..c2b7ba374 100644 --- a/ui/temboardui/model/orm.py +++ b/ui/temboardui/model/orm.py @@ -1,4 +1,6 @@ import datetime +import string +from secrets import choice from past.builtins import basestring from past.utils import old_div @@ -137,6 +139,12 @@ class ApiKeys(Model): cdate = Column(TIMESTAMP(timezone=True)) edate = Column(TIMESTAMP(timezone=True)) + _SECRET_LETTERS = string.ascii_letters + string.digits + "+/-_" + + @classmethod + def generate_secret(cls, length=40): + return "".join(choice(cls._SECRET_LETTERS) for _ in range(length)) + # See # https://docs.sqlalchemy.org/en/14/orm/queryguide.html#getting-orm-results-from-textual-and-core-statements diff --git a/ui/temboardui/plugins/monitoring/__init__.py b/ui/temboardui/plugins/monitoring/__init__.py index f98b5ca34..34ea3694c 100644 --- a/ui/temboardui/plugins/monitoring/__init__.py +++ b/ui/temboardui/plugins/monitoring/__init__.py @@ -21,7 +21,6 @@ import logging import os -import shutil from datetime import datetime, timedelta try: @@ -49,7 +48,6 @@ from ...core import refresh_discover from ...model import Session from ...toolkit import taskmanager -from ...toolkit import validators as v from ...toolkit.configuration import OptionSpec from ...toolkit.errors import UserError from .alerting import check_specs @@ -75,11 +73,7 @@ class MonitoringPlugin: s = "monitoring" - prometheus = shutil.which("prometheus") - options_specs = [ - OptionSpec(s, "collect_max_duration", default=30, validator=int), - OptionSpec(s, "prometheus", default=prometheus, validator=v.file_), - ] + options_specs = [OptionSpec(s, "collect_max_duration", default=30, validator=int)] def __init__(self, app): self.app = app diff --git a/ui/temboardui/prometheus.py b/ui/temboardui/prometheus.py new file mode 100644 index 000000000..7cf034f2a --- /dev/null +++ b/ui/temboardui/prometheus.py @@ -0,0 +1,256 @@ +# Bridge temBoard and Prometheus +# +# - Configure Prometheus with a dedicated API key. +# - Refresh prometheus configuration on SIGHUP and every 5 minutes. +# +import glob +import logging +import os +import re +import signal +import subprocess +import time +from datetime import datetime, timedelta + +import jinja2 + +import temboardui + +from .model import Session, orm +from .toolkit import syncio + +logger = logging.getLogger(__name__) + + +class Prometheus: + name = "prometheus" + + def __init__(self, binpath, home): + self.binpath = binpath + self.home = home + + def __str__(self): + return self.name + + # For services.BackgroundManager + @property + def pidfile(self): + return f"{self.home}/{self}.pid" + + # For services.execute() + def setup(self): + os.chdir(self.home) + + @property + def command(self): + return [ + self.binpath, + f"--config.file={self.home}/prometheus.yml", + # 0.0.0.0 is reachable from dev grafana for testing purpose. + # Once we embed alertmanager, use localhost. + "--web.listen-address=0.0.0.0:8890", + "--log.level=debug", + "--storage.tsdb.retention.time=1h", + ] + + +class Manager(syncio.Service): + name = "prometheus manager" + + def __init__(self, app): + self.app = app + self.sighup = False + self.provisionner = Provisionner(self.home) + self.configuration_expiry = None + # Set by services.BackgroundManager + self.pid = None + + def trigger_reload(self): + logger.debug("Triggering Prometheus configuration reload. pid=%s", self.pid) + os.kill(self.pid, signal.SIGHUP) + + @property + def home(self): + return self.app.config.temboard.home + "/prometheus" + + # Interface for SignalMultiplexer + def sighup_handler(self, *_): + self.sighup = True + + # Interface for services.run + def setup(self, sgm, bg): + logger.info("Provisionning Prometheus in %s.", self.home) + logger.debug("Creating temBoard API key.") + session = Session() + key = ( + orm.ApiKeys.insert( + secret=orm.ApiKeys.generate_secret(), + comment="temboard prometheus access", + ) + .with_session(session) + .scalar() + ) + session.commit() + + self.provisionner.setup_global(key.secret) + self._setup_instances() + + self.background = bg + bg.add(Prometheus(self.app.config.monitoring.prometheus, self.home)) + sgm.register(bg) + + # Interface for syncio.Loop + def accept(self): + if self.configuration_expiry < datetime.utcnow(): + logger.debug("Prometheus configuration expired.") + self._setup_instances() + self.sighup = False + + if self.sighup: + self.sighup = False + self.app.reload() + self._setup_instances() + + time.sleep(1) + + def _setup_instances(self): + provisionner = Provisionner(self.home) + provisionner.inspect() + + session = Session() + for instance in session.execute(orm.Instances.all()): + # DEPRECATED: SQAlchemy 1.4+ returns a tuple of objects + if len(instance) == 1: + (instance,) = instance + + if not instance.discover: + logger.debug("Skipping unreached instance %s.", instance) + continue + provisionner.setup_instance(instance) + + provisionner.purge() + + self.configuration_expiry = datetime.utcnow() + timedelta(minutes=5) + if hasattr(self, "background"): + self.background.kill(signal.SIGHUP) + + +def find(binpath=None): + candidates = [ + binpath, + # Package path. + "/usr/lib/temboard/prometheus", + # Development path for CI. + "ui/build/bin/prometheus", + ] + for binpath in candidates: + if not binpath: + continue + if os.path.exists(binpath): + return binpath + + +def version(binpath): + if not binpath: + return "n/a" + out = subprocess.run([binpath, "--version"], stdout=subprocess.PIPE, check=True) + m = re.search(r"version (\d+\.\d+\.\d+)", out.stdout.decode()) + if not m: + logger.warning("Failed to parse Prometheus version from %s", out.stdout) + return "n/a" + return m.group(1) + + +class Provisionner: + def __init__(self, home): + self.home = home + self.spurious_files = [] + + def inspect(self): + self.spurious_files = glob.glob(self.home + "/instances.d/instance-*.yml") + + def setup_global(self, apikey): + os.makedirs(self.home + "/instances.d", exist_ok=True) + t = jinja2.Template( + source=PROMETHEUS_CONFIG_TEMPLATE, + trim_blocks=True, + lstrip_blocks=True, + undefined=jinja2.StrictUndefined, + keep_trailing_newline=True, + ) + conf = t.render( + apikey=apikey, + home=self.home, + now=datetime.utcnow(), + temboard_version=temboardui.__version__, + ) + with open(self.home + "/prometheus.yml", "w") as fo: + os.chmod(fo.name, 0o600) + fo.write(conf) + + def setup_instance(self, instance): + path = "{}/instances.d/instance-{}-{}.scrape.yml".format( + self.home, instance.agent_address, instance.agent_port + ) + logger.debug( + "Configuring Prometheus scaping. instance=%s file=%s", + instance, + os.path.basename(path), + ) + t = jinja2.Template( + source=INSTANCE_CONFIG_TEMPLATE, + trim_blocks=True, + lstrip_blocks=True, + undefined=jinja2.StrictUndefined, + keep_trailing_newline=True, + ) + conf = t.render( + instance=instance, + now=datetime.utcnow(), + temboard_version=temboardui.__version__, + ) + with open(path, "w") as fo: + os.chmod(fo.name, 0o600) + fo.write(conf) + if path in self.spurious_files: + self.spurious_files.remove(path) + return path + + def purge(self): + for path in self.spurious_files: + logger.debug("Purging configuration file %s.", os.path.basename(path)) + os.unlink(path) + self.spurious_files = [] + + +PROMETHEUS_CONFIG_TEMPLATE = """\ +# This file is generated by temBoard. Do not edit. +# +# Generation Date: {{ now }} +# temBoard Version: {{ temboard_version }} +# +scrape_configs: +- job_name: temboard + scrape_interval: 60s + scheme: http + authorization: + type: Bearer + credentials: "{{ apikey }}" + file_sd_configs: + - files: + - instances.d/instance-*.yml +""" + + +INSTANCE_CONFIG_TEMPLATE = """\ +# This file is generated by temBoard. Do not edit. +# +# Generation Date: {{ now }} +# temBoard Version: {{ temboard_version }} +# +- targets: [localhost:8888] + labels: + __metrics_path__: "/proxy/{{ instance.agent_address }}/{{ instance.agent_port }}/monitoring/metrics" + instance: "{{ instance.hostname }}:{{ instance.pg_port }}" + agent: "{{ instance.agent_address }}:{{ instance.agent_port }}" +""" # noqa diff --git a/ui/temboardui/toolkit/services.py b/ui/temboardui/toolkit/services.py index 6194e320a..710e71a21 100644 --- a/ui/temboardui/toolkit/services.py +++ b/ui/temboardui/toolkit/services.py @@ -26,6 +26,8 @@ # - pkill -ef 'temboard: web' -KILL: scheduler and worker pool should stop. # - pkill -ef 'temboard: scheduler' -KILL: scheduler is restarted # - pkill -ef 'temboard: web' -HUP : all process must reload configuration +# - pkill -ef 'temboard: prometheus' -KILL: prometheus is restarted. +# - pkill -e prometheus -KILL: prometheus is restarted. # - Tornado autoreload: all processes are restarted. # # Same for temBoard agent. @@ -42,7 +44,7 @@ def run(main, *backgrounds): - """Execute a main service object function in this process. + """Run a main service object function in this process. Handle signals for INT, TERM, HUP, CHLD and ALRM. Handle background services. @@ -61,17 +63,13 @@ def run(main, *backgrounds): sgm.register(LoopStopper(loop)) sgm.register(main) - # for tornado autoreload. See TornadoService._autoreload_hook. - main.background = bg = BackgroundManager(loop) + bg = BackgroundManager(loop) if backgrounds: for service in backgrounds: bg.add(service) sgm.register(bg) - if getattr(main, "perf", None): - sgm.register(main.perf) - - main.setup() + main.setup(sgm, bg) with sgm, bg: logger.debug("Entering %s loop.", main) loop.start() @@ -81,6 +79,25 @@ def run(main, *backgrounds): return 0 +def execute(service): + """Execute an external command in this process. + + Replace current Python program by a command. + """ + + if hasattr(service, "setup"): + service.setup() + + # forking from asyncio loop requires reset of wakeup_fd. + signal.set_wakeup_fd(-1) + for fd in range(3, os.sysconf("SC_OPEN_MAX")): + try: + os.close(fd) + except OSError: + pass + os.execvp(service.command[0], service.command) + + class LoopStopper: def __init__(self, loop): self.loop = loop @@ -110,9 +127,20 @@ def __init__(self, loop): def add(self, service): self.services[str(service)] = service + def __bool__(self): + return bool(self.services) + def __enter__(self): if not self.services: return + + self._read_pids() + + if self.pids: + logger.debug("Cleaning previous background services.") + self.kill() + if self.wait(): + raise Exception("Background services are still alive.") self.start() def __exit__(self, *a): @@ -132,17 +160,51 @@ def fork(self, service): pid = os.fork() if pid: # Parent process logger.debug("Background service started. service=%s pid=%d", service, pid) - self.pids[str(service)] = pid + self._save_pid(service, pid) return pid # Child process - os._exit(run(service)) + if hasattr(service, "pidfile"): + execute(service) + else: + os._exit(run(service)) + + def _read_pids(self): + for name, service in self.services.items(): + if not hasattr(service, "pidfile"): + continue + if not os.path.exists(service.pidfile): + continue + with open(service.pidfile) as fo: + self.pids[name] = int(fo.read().strip()) + logger.debug( + "Read pid from pidfile. service=%s pid=%d", name, self.pids[name] + ) + + def _save_pid(self, service, pid): + service.pid = pid + self.pids[str(service)] = pid + if hasattr(service, "pidfile"): + with open(service.pidfile, "w") as fo: + fo.write(str(pid)) + + def _drop_pid(self, name): + del self.pids[name] + s = self.services[name] + s.pid = None + if not hasattr(s, "pidfile"): + return + if os.path.exists(s.pidfile): + os.unlink(s.pidfile) def stop(self): self.stopping = True for name, pid in self.pids.items(): logger.debug("Terminating background service. service=%s pid=%d", name, pid) - os.kill(pid, signal.SIGTERM) + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + pass time.sleep(0.125) if self.wait(): self.kill() @@ -160,20 +222,25 @@ def wait(self, timeout=5, step=0.5): try: pid, status = os.waitpid(-1, os.WNOHANG) except ChildProcessError: - break + pass if pid: - del self.pids[name] + self._drop_pid(name) time.sleep(step) timeout -= step return bool(self.pids) - def kill(self): + def kill(self, sig=signal.SIGKILL): for name, pid in self.pids.items(): - logger.warning("Killing background service. service=%s pid=%s", name, pid) + logger.warning( + "Signaling background service. service=%s pid=%s signal=%s", + name, + pid, + sig, + ) try: - os.kill(pid, signal.SIGKILL) + os.kill(pid, sig) except ProcessLookupError: pass @@ -195,11 +262,9 @@ def sigchld_handler(self, *a): pass logger.warning( - "Background service dead. Restarting service=%s pid=%s", - name, - self.pids[name], + "Background service dead. Restarting service=%s pid=%s", name, pid ) - del self.pids[name] + self._drop_pid(name) self.start() diff --git a/ui/temboardui/toolkit/taskmanager.py b/ui/temboardui/toolkit/taskmanager.py index 314e3c50c..451262900 100644 --- a/ui/temboardui/toolkit/taskmanager.py +++ b/ui/temboardui/toolkit/taskmanager.py @@ -443,7 +443,6 @@ def __init__(self, app, task_queue, event_queue): self.event_queue = event_queue self.scheduler = None self.task_list_engine = None - # For services.run self.perf = PerfCounters.setup(service=self.name) @property @@ -451,10 +450,11 @@ def address(self): return os.path.join(self.app.config.temboard.home, ".tm.socket") # interface for services.run - def setup(self): + def setup(self, sgm, *_, **__): self.teardown() self.scheduler.setup() if self.perf: + sgm.register(self.perf) self.perf.run() # interface for syncio.Loop @@ -535,7 +535,6 @@ def __init__(self, task_queue, event_queue): self.task_queue = task_queue self.event_queue = event_queue self.workers = {} - # For service.run() self.perf = None def _abort_job(self, task_id): @@ -789,13 +788,15 @@ class WorkerPoolService(syncio.Service): def __init__(self, app, task_queue, event_queue): super().__init__(app) self.worker_pool = WorkerPool(task_queue, event_queue) - # For service.run self.perf = PerfCounters.setup(service=self.name) # interface for services.run - def setup(self): + def setup(self, sgm, *_, **__): self.worker_pool.perf = self.perf self.worker_pool.setup() + if self.perf: + sgm.register(self.perf) + self.perf.run() def teardown(self): logger.info("Aborting jobs.") diff --git a/ui/temboardui/version.py b/ui/temboardui/version.py index 2753753d0..406b61fae 100644 --- a/ui/temboardui/version.py +++ b/ui/temboardui/version.py @@ -1,7 +1,9 @@ +import logging import sys from platform import python_version __version__ = "9.0.dev0" +logger = logging.getLogger(__name__) # This output is parsed by tests/conftest.py::pytest_report_header. @@ -14,6 +16,7 @@ libpq %(libpq)s psycopg2 %(psycopg2)s SQLAlchemy %(sqlalchemy)s +Prometheus %(prometheus)s (%(prometheusbin)s) """ @@ -21,15 +24,18 @@ def format_version(): return VERSION_FMT % inspect_versions() -def inspect_versions(): +def inspect_versions(prometheusbin=None): import cryptography from psycopg2 import __version__ as psycopg2_version from sqlalchemy import __version__ as sqlalchemy_version from tornado import version as tornado_version + from . import prometheus from .toolkit.versions import format_pq_version, read_distinfo, read_libpq_version distinfos = read_distinfo() + prometheusbin = prometheus.find(prometheusbin) + prometheus = prometheus.version(prometheusbin) return dict( cryptography=cryptography.__version__, @@ -43,4 +49,6 @@ def inspect_versions(): temboard=__version__, temboardbin=sys.argv[0], tornado=tornado_version, + prometheus=prometheus, + prometheusbin=prometheusbin or "n/a", )