diff --git a/core/emperor.c b/core/emperor.c index cd99e75500..3c790f5b35 100644 --- a/core/emperor.c +++ b/core/emperor.c @@ -2528,7 +2528,12 @@ static void emperor_send_stats(int fd) { } if (uwsgi.stats_http) { - if (uwsgi_send_http_stats(client_fd)) { + enum uwsgi_stats_format fmt; + if (uwsgi_stats_read_request(client_fd, &fmt)) { + close(client_fd); + return; + } + if (uwsgi_stats_send_http_header(client_fd, UWSGI_STATS_FORMAT_JSON)) { close(client_fd); return; } diff --git a/core/stats.c b/core/stats.c index 31ddd65239..50fe7e8dc3 100644 --- a/core/stats.c +++ b/core/stats.c @@ -365,17 +365,33 @@ void uwsgi_send_stats(int fd, struct uwsgi_stats *(*func) (void)) { return; } + enum uwsgi_stats_format fmt = UWSGI_STATS_FORMAT_JSON; if (uwsgi.stats_http) { - if (uwsgi_send_http_stats(client_fd)) { + if (uwsgi_stats_read_request(client_fd, &fmt)) { close(client_fd); return; } } - struct uwsgi_stats *us = func(); + struct uwsgi_stats *us; + switch (fmt) { + case UWSGI_STATS_FORMAT_PROMETHEUS: + us = uwsgi_master_generate_stats_prometheus(); + break; + default: + us = func(); + break; + } + if (!us) goto end; + if (uwsgi.stats_http) { + if (uwsgi_stats_send_http_header(client_fd, fmt)) { + goto end0; + } + } + size_t remains = us->pos; off_t pos = 0; while (remains > 0) { diff --git a/core/stats_prometheus.c b/core/stats_prometheus.c new file mode 100644 index 0000000000..989fa3e3ec --- /dev/null +++ b/core/stats_prometheus.c @@ -0,0 +1,733 @@ +#include "uwsgi.h" + +extern struct uwsgi_server uwsgi; + +/* + * Prometheus text exposition format generator for uWSGI stats. + * + * Metric names and semantics are aligned with timonwong/uwsgi_exporter + * for drop-in compatibility. + */ + +/* Helper: ensure buffer has room for 'needed' more bytes */ +static int prom_ensure(struct uwsgi_stats *us, size_t needed) { + while (us->pos + needed >= us->size) { + size_t new_size = us->size + us->chunk; + char *new_base = realloc(us->base, new_size); + if (!new_base) return -1; + us->base = new_base; + us->size = new_size; + } + return 0; +} + +/* Append raw string to buffer */ +static int prom_append(struct uwsgi_stats *us, const char *str) { + size_t len = strlen(str); + if (prom_ensure(us, len)) return -1; + memcpy(us->base + us->pos, str, len); + us->pos += len; + return 0; +} + +/* Append a formatted string to buffer */ +static int prom_printf(struct uwsgi_stats *us, const char *fmt, ...) { + char buf[4096]; + va_list ap; + va_start(ap, fmt); + int n = vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + if (n < 0) return -1; + if ((size_t)n >= sizeof(buf)) return -1; + return prom_append(us, buf); +} + +/* Escape a label value per Prometheus spec: \ -> \\, " -> \", \n -> \n */ +static int prom_append_escaped(struct uwsgi_stats *us, const char *val, size_t len) { + /* Worst case: every char needs escaping -> 2x */ + if (prom_ensure(us, len * 2 + 1)) return -1; + size_t i; + for (i = 0; i < len; i++) { + char c = val[i]; + if (c == '\\') { + us->base[us->pos++] = '\\'; + us->base[us->pos++] = '\\'; + } else if (c == '"') { + us->base[us->pos++] = '\\'; + us->base[us->pos++] = '"'; + } else if (c == '\n') { + us->base[us->pos++] = '\\'; + us->base[us->pos++] = 'n'; + } else { + us->base[us->pos++] = c; + } + } + return 0; +} + +/* Write HELP + TYPE header for a metric */ +static int prom_header(struct uwsgi_stats *us, const char *name, const char *type, const char *help) { + if (prom_printf(us, "# HELP %s %s\n# TYPE %s %s\n", name, help, name, type)) return -1; + return 0; +} + +/* Write a metric line with no labels: name value\n */ +static int prom_metric(struct uwsgi_stats *us, const char *name, const char *type, const char *help, unsigned long long val) { + if (prom_header(us, name, type, help)) return -1; + if (prom_printf(us, "%s %llu\n", name, val)) return -1; + return 0; +} + +/* --- Sanitize metric name for Prometheus: must match [a-zA-Z_:][a-zA-Z0-9_:]* --- */ +static void sanitize_metric_name(char *dst, const char *src, size_t dst_size) { + size_t i; + size_t slen = strlen(src); + if (dst_size == 0) return; + if (slen >= dst_size) slen = dst_size - 1; + for (i = 0; i < slen; i++) { + char c = src[i]; + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '_' || c == ':') { + dst[i] = c; + } else { + dst[i] = '_'; + } + } + dst[i] = '\0'; + /* Prometheus names must not start with a digit */ + if (i > 0 && dst[0] >= '0' && dst[0] <= '9') { + /* Shift right and prepend underscore if there's room */ + if (i + 1 < dst_size) { + memmove(dst + 1, dst, i + 1); + dst[0] = '_'; + } else { + dst[0] = '_'; + } + } +} + + +/* Check if this socket name+proto was already seen earlier in the list */ +static int prom_socket_is_duplicate(struct uwsgi_socket *head, struct uwsgi_socket *current) { + const char *cur_proto = current->proto_name ? current->proto_name : "uwsgi"; + struct uwsgi_socket *s = head; + while (s != current) { + const char *s_proto = s->proto_name ? s->proto_name : "uwsgi"; + if (!strcmp(s->name, current->name) && !strcmp(s_proto, cur_proto)) { + return 1; + } + s = s->next; + } + return 0; +} + +struct uwsgi_stats *uwsgi_master_generate_stats_prometheus(void) { + int i; + + struct uwsgi_stats *us = uwsgi_malloc(sizeof(struct uwsgi_stats)); + us->chunk = 8192; + us->size = us->chunk; + us->base = uwsgi_malloc(us->size); + us->pos = 0; + us->tabs = 0; + us->dirty = 0; + us->minified = 1; + + /* ============================================================ + * 2.1 Global metrics + * ============================================================ */ + +#ifdef __linux__ + if (prom_metric(us, "uwsgi_listen_queue_length", "gauge", + "Length of listen queue.", + (unsigned long long)uwsgi.shared->backlog)) + goto end; + if (prom_metric(us, "uwsgi_listen_queue_errors", "gauge", + "Number of listen queue errors.", + (unsigned long long)uwsgi.shared->backlog_errors)) + goto end; +#endif + + int signal_queue = 0; + if (ioctl(uwsgi.shared->worker_signal_pipe[1], FIONREAD, &signal_queue)) { + uwsgi_error("uwsgi_master_generate_stats_prometheus() -> ioctl()\n"); + } + if (prom_metric(us, "uwsgi_signal_queue_length", "gauge", + "Length of signal queue.", + (unsigned long long)signal_queue)) + goto end; + + /* Count workers with id != 0 */ + { + int worker_count = 0; + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i + 1].id != 0) worker_count++; + } + if (prom_metric(us, "uwsgi_workers", "gauge", + "Number of workers.", + (unsigned long long)worker_count)) + goto end; + } + + /* ============================================================ + * 2.2 Socket metrics (labels: name, proto) + * ============================================================ */ + + { + struct uwsgi_socket *uwsgi_sock = uwsgi.sockets; + + /* socket_queue_length */ + if (prom_header(us, "uwsgi_socket_queue_length", "gauge", "Length of socket queue.")) goto end; + uwsgi_sock = uwsgi.sockets; + while (uwsgi_sock) { + if (prom_socket_is_duplicate(uwsgi.sockets, uwsgi_sock)) { uwsgi_sock = uwsgi_sock->next; continue; } + const char *proto = uwsgi_sock->proto_name ? uwsgi_sock->proto_name : "uwsgi"; + if (prom_printf(us, "uwsgi_socket_queue_length{name=\"")) goto end; + if (prom_append_escaped(us, uwsgi_sock->name, strlen(uwsgi_sock->name))) goto end; + if (prom_printf(us, "\",proto=\"%s\"} %llu\n", proto, (unsigned long long)uwsgi_sock->queue)) goto end; + uwsgi_sock = uwsgi_sock->next; + } + + /* socket_max_queue_length */ + if (prom_header(us, "uwsgi_socket_max_queue_length", "gauge", "Max length of socket queue.")) goto end; + uwsgi_sock = uwsgi.sockets; + while (uwsgi_sock) { + if (prom_socket_is_duplicate(uwsgi.sockets, uwsgi_sock)) { uwsgi_sock = uwsgi_sock->next; continue; } + const char *proto = uwsgi_sock->proto_name ? uwsgi_sock->proto_name : "uwsgi"; + if (prom_printf(us, "uwsgi_socket_max_queue_length{name=\"")) goto end; + if (prom_append_escaped(us, uwsgi_sock->name, strlen(uwsgi_sock->name))) goto end; + if (prom_printf(us, "\",proto=\"%s\"} %llu\n", proto, (unsigned long long)uwsgi_sock->max_queue)) goto end; + uwsgi_sock = uwsgi_sock->next; + } + + /* socket_shared */ + if (prom_header(us, "uwsgi_socket_shared", "gauge", "Is shared socket?")) goto end; + uwsgi_sock = uwsgi.sockets; + while (uwsgi_sock) { + if (prom_socket_is_duplicate(uwsgi.sockets, uwsgi_sock)) { uwsgi_sock = uwsgi_sock->next; continue; } + const char *proto = uwsgi_sock->proto_name ? uwsgi_sock->proto_name : "uwsgi"; + if (prom_printf(us, "uwsgi_socket_shared{name=\"")) goto end; + if (prom_append_escaped(us, uwsgi_sock->name, strlen(uwsgi_sock->name))) goto end; + if (prom_printf(us, "\",proto=\"%s\"} %llu\n", proto, (unsigned long long)uwsgi_sock->shared)) goto end; + uwsgi_sock = uwsgi_sock->next; + } + + /* socket_can_offload */ + if (prom_header(us, "uwsgi_socket_can_offload", "gauge", "Can socket offload?")) goto end; + uwsgi_sock = uwsgi.sockets; + while (uwsgi_sock) { + if (prom_socket_is_duplicate(uwsgi.sockets, uwsgi_sock)) { uwsgi_sock = uwsgi_sock->next; continue; } + const char *proto = uwsgi_sock->proto_name ? uwsgi_sock->proto_name : "uwsgi"; + if (prom_printf(us, "uwsgi_socket_can_offload{name=\"")) goto end; + if (prom_append_escaped(us, uwsgi_sock->name, strlen(uwsgi_sock->name))) goto end; + if (prom_printf(us, "\",proto=\"%s\"} %llu\n", proto, (unsigned long long)uwsgi_sock->can_offload)) goto end; + uwsgi_sock = uwsgi_sock->next; + } + } + + /* ============================================================ + * 2.3 Worker metrics (label: worker_id) + * ============================================================ */ + +#define WORKER_METRIC_HEADER(metric_name, metric_type, metric_help) \ + if (prom_header(us, metric_name, metric_type, metric_help)) goto end; + +#define WORKER_METRIC_ULL(metric_name, value) \ + if (prom_printf(us, "%s{worker_id=\"%d\"} %llu\n", metric_name, uwsgi.workers[i+1].id, (unsigned long long)(value))) goto end; + +#define WORKER_METRIC_FLOAT(metric_name, value) \ + if (prom_printf(us, "%s{worker_id=\"%d\"} %.6f\n", metric_name, uwsgi.workers[i+1].id, (double)(value))) goto end; + + /* uwsgi_worker_accepting */ + WORKER_METRIC_HEADER("uwsgi_worker_accepting", "gauge", "Is this worker accepting requests?") + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + WORKER_METRIC_ULL("uwsgi_worker_accepting", uwsgi.workers[i+1].accepting) + } + + /* uwsgi_worker_requests_total */ + WORKER_METRIC_HEADER("uwsgi_worker_requests_total", "counter", "Total number of requests.") + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + WORKER_METRIC_ULL("uwsgi_worker_requests_total", uwsgi.workers[i+1].requests) + } + + /* uwsgi_worker_delta_requests */ + WORKER_METRIC_HEADER("uwsgi_worker_delta_requests", "gauge", "Number of delta requests.") + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + WORKER_METRIC_ULL("uwsgi_worker_delta_requests", uwsgi.workers[i+1].delta_requests) + } + + /* uwsgi_worker_exceptions_total */ + WORKER_METRIC_HEADER("uwsgi_worker_exceptions_total", "counter", "Total number of exceptions.") + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + WORKER_METRIC_ULL("uwsgi_worker_exceptions_total", uwsgi_worker_exceptions(i+1)) + } + + /* uwsgi_worker_harakiri_count_total */ + WORKER_METRIC_HEADER("uwsgi_worker_harakiri_count_total", "counter", "Total number of harakiri count.") + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + WORKER_METRIC_ULL("uwsgi_worker_harakiri_count_total", uwsgi.workers[i+1].harakiri_count) + } + + /* uwsgi_worker_signals_total */ + WORKER_METRIC_HEADER("uwsgi_worker_signals_total", "counter", "Total number of signals.") + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + WORKER_METRIC_ULL("uwsgi_worker_signals_total", uwsgi.workers[i+1].signals) + } + + /* uwsgi_worker_signal_queue_length */ + WORKER_METRIC_HEADER("uwsgi_worker_signal_queue_length", "gauge", "Length of signal queue.") + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + int wq = 0; + if (ioctl(uwsgi.workers[i+1].signal_pipe[1], FIONREAD, &wq)) { + uwsgi_error("uwsgi_master_generate_stats_prometheus() -> ioctl()\n"); + } + WORKER_METRIC_ULL("uwsgi_worker_signal_queue_length", wq) + } + + /* uwsgi_worker_rss_bytes */ + WORKER_METRIC_HEADER("uwsgi_worker_rss_bytes", "gauge", "Worker RSS bytes.") + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + WORKER_METRIC_ULL("uwsgi_worker_rss_bytes", uwsgi.workers[i+1].rss_size) + } + + /* uwsgi_worker_vsz_bytes */ + WORKER_METRIC_HEADER("uwsgi_worker_vsz_bytes", "gauge", "Worker VSZ bytes.") + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + WORKER_METRIC_ULL("uwsgi_worker_vsz_bytes", uwsgi.workers[i+1].vsz_size) + } + + /* uwsgi_worker_running_time_seconds (microseconds -> seconds) */ + WORKER_METRIC_HEADER("uwsgi_worker_running_time_seconds", "gauge", "Worker running time in seconds.") + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + WORKER_METRIC_FLOAT("uwsgi_worker_running_time_seconds", (double)uwsgi.workers[i+1].running_time / 1000000.0) + } + + /* uwsgi_worker_last_spawn_time_seconds */ + WORKER_METRIC_HEADER("uwsgi_worker_last_spawn_time_seconds", "gauge", "Last spawn time in seconds since epoch.") + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + WORKER_METRIC_ULL("uwsgi_worker_last_spawn_time_seconds", uwsgi.workers[i+1].last_spawn) + } + + /* uwsgi_worker_average_response_time_seconds (microseconds -> seconds) */ + WORKER_METRIC_HEADER("uwsgi_worker_average_response_time_seconds", "gauge", "Average response time in seconds.") + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + WORKER_METRIC_FLOAT("uwsgi_worker_average_response_time_seconds", (double)uwsgi.workers[i+1].avg_response_time / 1000000.0) + } + + /* uwsgi_worker_apps */ + WORKER_METRIC_HEADER("uwsgi_worker_apps", "gauge", "Number of apps.") + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + WORKER_METRIC_ULL("uwsgi_worker_apps", uwsgi.workers[i+1].apps_cnt) + } + + /* uwsgi_worker_cores */ + WORKER_METRIC_HEADER("uwsgi_worker_cores", "gauge", "Number of cores.") + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + WORKER_METRIC_ULL("uwsgi_worker_cores", uwsgi.cores) + } + + /* uwsgi_worker_busy */ + WORKER_METRIC_HEADER("uwsgi_worker_busy", "gauge", "Is core in busy?") + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + int is_busy = 0; + /* Match canonical JSON status logic from master_utils.c: + cheaped -> "cheap", suspended && !busy -> "pause", + sig -> "sig", busy -> "busy", else -> "idle" */ + if (!uwsgi.workers[i+1].cheaped && + !(uwsgi.workers[i+1].suspended && !uwsgi_worker_is_busy(i+1)) && + !uwsgi.workers[i+1].sig && + uwsgi_worker_is_busy(i+1)) is_busy = 1; + WORKER_METRIC_ULL("uwsgi_worker_busy", is_busy) + } + + /* uwsgi_worker_idle */ + WORKER_METRIC_HEADER("uwsgi_worker_idle", "gauge", "Is core in idle?") + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + int is_idle = 0; + /* Match canonical JSON status logic: idle only when not cheaped, + not paused, not signaled, and not busy */ + if (!uwsgi.workers[i+1].cheaped && + !(uwsgi.workers[i+1].suspended && !uwsgi_worker_is_busy(i+1)) && + !uwsgi.workers[i+1].sig && + !uwsgi_worker_is_busy(i+1)) is_idle = 1; + WORKER_METRIC_ULL("uwsgi_worker_idle", is_idle) + } + + /* uwsgi_worker_cheap */ + WORKER_METRIC_HEADER("uwsgi_worker_cheap", "gauge", "Is core in cheap mode?") + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + WORKER_METRIC_ULL("uwsgi_worker_cheap", uwsgi.workers[i+1].cheaped ? 1 : 0) + } + + /* uwsgi_worker_respawn_count_total */ + WORKER_METRIC_HEADER("uwsgi_worker_respawn_count_total", "counter", "Total number of respawn count.") + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + WORKER_METRIC_ULL("uwsgi_worker_respawn_count_total", uwsgi.workers[i+1].respawn_count) + } + + /* uwsgi_worker_transmitted_bytes_total */ + WORKER_METRIC_HEADER("uwsgi_worker_transmitted_bytes_total", "counter", "Worker transmitted bytes.") + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + WORKER_METRIC_ULL("uwsgi_worker_transmitted_bytes_total", uwsgi.workers[i+1].tx) + } + +#undef WORKER_METRIC_HEADER +#undef WORKER_METRIC_ULL +#undef WORKER_METRIC_FLOAT + + /* ============================================================ + * 2.4 Worker app metrics (labels: worker_id, app_id, mountpoint, chdir) + * ============================================================ */ + + /* uwsgi_worker_app_startup_time_seconds */ + if (prom_header(us, "uwsgi_worker_app_startup_time_seconds", "gauge", "How long this app took to start.")) goto end; + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + int j; + for (j = 0; j < uwsgi.workers[i+1].apps_cnt; j++) { + struct uwsgi_app *ua = &uwsgi.workers[i+1].apps[j]; + if (prom_printf(us, "uwsgi_worker_app_startup_time_seconds{worker_id=\"%d\",app_id=\"%d\",mountpoint=\"", + uwsgi.workers[i+1].id, j)) goto end; + if (prom_append_escaped(us, ua->mountpoint, ua->mountpoint_len)) goto end; + if (prom_append(us, "\",chdir=\"")) goto end; + if (prom_append_escaped(us, ua->chdir, strlen(ua->chdir))) goto end; + if (prom_printf(us, "\"} %llu\n", (unsigned long long)ua->startup_time)) goto end; + } + } + + /* uwsgi_worker_app_requests_total */ + if (prom_header(us, "uwsgi_worker_app_requests_total", "counter", "Total number of requests.")) goto end; + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + int j; + for (j = 0; j < uwsgi.workers[i+1].apps_cnt; j++) { + struct uwsgi_app *ua = &uwsgi.workers[i+1].apps[j]; + if (prom_printf(us, "uwsgi_worker_app_requests_total{worker_id=\"%d\",app_id=\"%d\",mountpoint=\"", + uwsgi.workers[i+1].id, j)) goto end; + if (prom_append_escaped(us, ua->mountpoint, ua->mountpoint_len)) goto end; + if (prom_append(us, "\",chdir=\"")) goto end; + if (prom_append_escaped(us, ua->chdir, strlen(ua->chdir))) goto end; + if (prom_printf(us, "\"} %llu\n", (unsigned long long)ua->requests)) goto end; + } + } + + /* uwsgi_worker_app_exceptions_total */ + if (prom_header(us, "uwsgi_worker_app_exceptions_total", "counter", "Total number of exceptions.")) goto end; + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + int j; + for (j = 0; j < uwsgi.workers[i+1].apps_cnt; j++) { + struct uwsgi_app *ua = &uwsgi.workers[i+1].apps[j]; + if (prom_printf(us, "uwsgi_worker_app_exceptions_total{worker_id=\"%d\",app_id=\"%d\",mountpoint=\"", + uwsgi.workers[i+1].id, j)) goto end; + if (prom_append_escaped(us, ua->mountpoint, ua->mountpoint_len)) goto end; + if (prom_append(us, "\",chdir=\"")) goto end; + if (prom_append_escaped(us, ua->chdir, strlen(ua->chdir))) goto end; + if (prom_printf(us, "\"} %llu\n", (unsigned long long)ua->exceptions)) goto end; + } + } + + /* ============================================================ + * 2.5 Worker core metrics (labels: worker_id, core_id) + * Skip if --stats-no-cores + * ============================================================ */ + + if (!uwsgi.stats_no_cores) { + + /* uwsgi_worker_core_busy */ + if (prom_header(us, "uwsgi_worker_core_busy", "gauge", "Is core busy.")) goto end; + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + int j; + for (j = 0; j < uwsgi.cores; j++) { + struct uwsgi_core *uc = &uwsgi.workers[i+1].cores[j]; + if (prom_printf(us, "uwsgi_worker_core_busy{worker_id=\"%d\",core_id=\"%d\"} %llu\n", + uwsgi.workers[i+1].id, j, (unsigned long long)uc->in_request)) goto end; + } + } + + /* uwsgi_worker_core_requests_total */ + if (prom_header(us, "uwsgi_worker_core_requests_total", "counter", "Total number of requests.")) goto end; + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + int j; + for (j = 0; j < uwsgi.cores; j++) { + struct uwsgi_core *uc = &uwsgi.workers[i+1].cores[j]; + if (prom_printf(us, "uwsgi_worker_core_requests_total{worker_id=\"%d\",core_id=\"%d\"} %llu\n", + uwsgi.workers[i+1].id, j, (unsigned long long)uc->requests)) goto end; + } + } + + /* uwsgi_worker_core_static_requests_total */ + if (prom_header(us, "uwsgi_worker_core_static_requests_total", "counter", "Total number of static requests.")) goto end; + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + int j; + for (j = 0; j < uwsgi.cores; j++) { + struct uwsgi_core *uc = &uwsgi.workers[i+1].cores[j]; + if (prom_printf(us, "uwsgi_worker_core_static_requests_total{worker_id=\"%d\",core_id=\"%d\"} %llu\n", + uwsgi.workers[i+1].id, j, (unsigned long long)uc->static_requests)) goto end; + } + } + + /* uwsgi_worker_core_routed_requests_total */ + if (prom_header(us, "uwsgi_worker_core_routed_requests_total", "counter", "Total number of routed requests.")) goto end; + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + int j; + for (j = 0; j < uwsgi.cores; j++) { + struct uwsgi_core *uc = &uwsgi.workers[i+1].cores[j]; + if (prom_printf(us, "uwsgi_worker_core_routed_requests_total{worker_id=\"%d\",core_id=\"%d\"} %llu\n", + uwsgi.workers[i+1].id, j, (unsigned long long)uc->routed_requests)) goto end; + } + } + + /* uwsgi_worker_core_offloaded_requests_total */ + if (prom_header(us, "uwsgi_worker_core_offloaded_requests_total", "counter", "Total number of offloaded requests.")) goto end; + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + int j; + for (j = 0; j < uwsgi.cores; j++) { + struct uwsgi_core *uc = &uwsgi.workers[i+1].cores[j]; + if (prom_printf(us, "uwsgi_worker_core_offloaded_requests_total{worker_id=\"%d\",core_id=\"%d\"} %llu\n", + uwsgi.workers[i+1].id, j, (unsigned long long)uc->offloaded_requests)) goto end; + } + } + + /* uwsgi_worker_core_write_errors_total */ + if (prom_header(us, "uwsgi_worker_core_write_errors_total", "counter", "Total number of write errors.")) goto end; + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + int j; + for (j = 0; j < uwsgi.cores; j++) { + struct uwsgi_core *uc = &uwsgi.workers[i+1].cores[j]; + if (prom_printf(us, "uwsgi_worker_core_write_errors_total{worker_id=\"%d\",core_id=\"%d\"} %llu\n", + uwsgi.workers[i+1].id, j, (unsigned long long)uc->write_errors)) goto end; + } + } + + /* uwsgi_worker_core_read_errors_total */ + if (prom_header(us, "uwsgi_worker_core_read_errors_total", "counter", "Total number of read errors.")) goto end; + for (i = 0; i < uwsgi.numproc; i++) { + if (uwsgi.workers[i+1].id == 0) continue; + int j; + for (j = 0; j < uwsgi.cores; j++) { + struct uwsgi_core *uc = &uwsgi.workers[i+1].cores[j]; + if (prom_printf(us, "uwsgi_worker_core_read_errors_total{worker_id=\"%d\",core_id=\"%d\"} %llu\n", + uwsgi.workers[i+1].id, j, (unsigned long long)uc->read_errors)) goto end; + } + } + } + + /* ============================================================ + * 2.6 Cache metrics (label: name) + * ============================================================ */ + + if (uwsgi.caches) { + struct uwsgi_cache *uc; + + /* uwsgi_cache_hits */ + if (prom_header(us, "uwsgi_cache_hits", "counter", "Total number of hits.")) goto end; + uc = uwsgi.caches; + while (uc) { + const char *name = uc->name ? uc->name : "default"; + if (prom_printf(us, "uwsgi_cache_hits{name=\"")) goto end; + if (prom_append_escaped(us, name, strlen(name))) goto end; + if (prom_printf(us, "\"} %llu\n", (unsigned long long)uc->hits)) goto end; + uc = uc->next; + } + + /* uwsgi_cache_misses */ + if (prom_header(us, "uwsgi_cache_misses", "counter", "Total number of misses.")) goto end; + uc = uwsgi.caches; + while (uc) { + const char *name = uc->name ? uc->name : "default"; + if (prom_printf(us, "uwsgi_cache_misses{name=\"")) goto end; + if (prom_append_escaped(us, name, strlen(name))) goto end; + if (prom_printf(us, "\"} %llu\n", (unsigned long long)uc->miss)) goto end; + uc = uc->next; + } + + /* uwsgi_cache_full */ + if (prom_header(us, "uwsgi_cache_full", "counter", "Total number of times cache full was hit.")) goto end; + uc = uwsgi.caches; + while (uc) { + const char *name = uc->name ? uc->name : "default"; + if (prom_printf(us, "uwsgi_cache_full{name=\"")) goto end; + if (prom_append_escaped(us, name, strlen(name))) goto end; + if (prom_printf(us, "\"} %llu\n", (unsigned long long)uc->full)) goto end; + uc = uc->next; + } + + /* uwsgi_cache_items */ + if (prom_header(us, "uwsgi_cache_items", "gauge", "Items in cache.")) goto end; + uc = uwsgi.caches; + while (uc) { + const char *name = uc->name ? uc->name : "default"; + if (prom_printf(us, "uwsgi_cache_items{name=\"")) goto end; + if (prom_append_escaped(us, name, strlen(name))) goto end; + if (prom_printf(us, "\"} %llu\n", (unsigned long long)uc->n_items)) goto end; + uc = uc->next; + } + + /* uwsgi_cache_max_items */ + if (prom_header(us, "uwsgi_cache_max_items", "gauge", "Max items for this cache.")) goto end; + uc = uwsgi.caches; + while (uc) { + const char *name = uc->name ? uc->name : "default"; + if (prom_printf(us, "uwsgi_cache_max_items{name=\"")) goto end; + if (prom_append_escaped(us, name, strlen(name))) goto end; + if (prom_printf(us, "\"} %llu\n", (unsigned long long)uc->max_items)) goto end; + uc = uc->next; + } + } + + /* ============================================================ + * 2.7 Custom metrics from uWSGI metrics subsystem + * ============================================================ */ + + if (uwsgi.has_metrics && !uwsgi.stats_no_metrics) { + uwsgi_rlock(uwsgi.metrics_lock); + + /* First pass: count metrics to size the seen-names array */ + int num_custom = 0; + struct uwsgi_metric *um = uwsgi.metrics; + while (um) { + if (um->type != UWSGI_METRIC_ALIAS) num_custom++; + um = um->next; + } + + /* Track emitted sanitized names to detect collisions from + sanitization (e.g. "foo.bar" and "foo_bar" both become + "uwsgi_custom_foo_bar"). Skip duplicates to avoid invalid + Prometheus output with repeated family declarations. */ + char **seen_names = NULL; + int seen_count = 0; + if (num_custom > 0) { + seen_names = (char **)malloc(sizeof(char *) * num_custom); + if (!seen_names) { + uwsgi_rwunlock(uwsgi.metrics_lock); + goto end; + } + } + + um = uwsgi.metrics; + while (um) { + /* Skip aliases to avoid duplicates */ + if (um->type == UWSGI_METRIC_ALIAS) { + um = um->next; + continue; + } + + int64_t um_val = *um->value; + + /* Sanitize metric name */ + char sanitized[256]; + char prefixed[512]; + sanitize_metric_name(sanitized, um->name, sizeof(sanitized)); + int plen = snprintf(prefixed, sizeof(prefixed), "uwsgi_custom_%s", sanitized); + if (plen < 0 || (size_t)plen >= sizeof(prefixed)) { + um = um->next; + continue; /* skip metrics with names too long to prefix */ + } + + /* Check for sanitized name collision */ + int is_dup = 0; + for (int si = 0; si < seen_count; si++) { + if (!strcmp(seen_names[si], prefixed)) { + is_dup = 1; + break; + } + } + if (is_dup) { + um = um->next; + continue; + } + seen_names[seen_count] = strdup(prefixed); + if (!seen_names[seen_count]) { + um = um->next; + continue; /* skip metric if we cannot track its name */ + } + seen_count++; + + const char *prom_type; + switch (um->type) { + case UWSGI_METRIC_COUNTER: + prom_type = "counter"; + break; + case UWSGI_METRIC_GAUGE: + prom_type = "gauge"; + break; + case UWSGI_METRIC_ABSOLUTE: + prom_type = "gauge"; + break; + default: + prom_type = "gauge"; + break; + } + + /* Truncate help text to avoid overflowing prom_printf's + 4096-byte buffer when metric names are very long */ + char help_buf[1024]; + size_t name_len = strlen(um->name); + if (name_len >= sizeof(help_buf)) { + memcpy(help_buf, um->name, sizeof(help_buf) - 4); + help_buf[sizeof(help_buf) - 4] = '.'; + help_buf[sizeof(help_buf) - 3] = '.'; + help_buf[sizeof(help_buf) - 2] = '.'; + help_buf[sizeof(help_buf) - 1] = '\0'; + } else { + memcpy(help_buf, um->name, name_len + 1); + } + + if (prom_header(us, prefixed, prom_type, help_buf)) { + for (int si = 0; si < seen_count; si++) free(seen_names[si]); + free(seen_names); + uwsgi_rwunlock(uwsgi.metrics_lock); + goto end; + } + + if (prom_printf(us, "%s %lld\n", prefixed, (long long)um_val)) { + for (int si = 0; si < seen_count; si++) free(seen_names[si]); + free(seen_names); + uwsgi_rwunlock(uwsgi.metrics_lock); + goto end; + } + + um = um->next; + } + for (int si = 0; si < seen_count; si++) free(seen_names[si]); + free(seen_names); + uwsgi_rwunlock(uwsgi.metrics_lock); + } + + /* Null-terminate the buffer */ + if (prom_ensure(us, 1)) goto end; + us->base[us->pos] = '\0'; + + return us; + +end: + free(us->base); + free(us); + return NULL; +} diff --git a/core/utils.c b/core/utils.c index c944f64781..f497238d69 100644 --- a/core/utils.c +++ b/core/utils.c @@ -3988,16 +3988,45 @@ int uwsgi_kvlist_parse(char *src, size_t len, char list_separator, int kv_separa return 0; } -int uwsgi_send_http_stats(int fd) { +int uwsgi_stats_read_request(int fd, enum uwsgi_stats_format *fmt) { char buf[4096]; + *fmt = UWSGI_STATS_FORMAT_JSON; + int ret = uwsgi_waitfd(fd, uwsgi.socket_timeout); if (ret <= 0) return -1; - if (read(fd, buf, 4096) <= 0) + ssize_t rlen = read(fd, buf, 4096 - 1); + if (rlen <= 0) return -1; + buf[rlen] = '\0'; + + // parse request path: GET HTTP/... + if (!strncmp(buf, "GET ", 4)) { + char *path_start = buf + 4; + char *path_end = strchr(path_start, ' '); + if (path_end) { + size_t path_len = path_end - path_start; + char *query = memchr(path_start, '?', path_len); + if (query) path_len = query - path_start; + char *prom_path = uwsgi.stats_prometheus_path ? uwsgi.stats_prometheus_path : "/metrics"; + size_t prom_len = strlen(prom_path); + if (path_len == prom_len && !strncmp(path_start, prom_path, prom_len)) { + *fmt = UWSGI_STATS_FORMAT_PROMETHEUS; + } + } + } + + return 0; +} + +int uwsgi_stats_send_http_header(int fd, enum uwsgi_stats_format fmt) { + + char *content_type = (fmt == UWSGI_STATS_FORMAT_PROMETHEUS) + ? "Content-Type: text/plain; version=0.0.4; charset=utf-8\r\n" + : "Content-Type: application/json\r\n"; struct uwsgi_buffer *ub = uwsgi_buffer_new(uwsgi.page_size); if (!ub) @@ -4009,7 +4038,7 @@ int uwsgi_send_http_stats(int fd) { goto error; if (uwsgi_buffer_append(ub, "Access-Control-Allow-Origin: *\r\n", 32)) goto error; - if (uwsgi_buffer_append(ub, "Content-Type: application/json\r\n", 32)) + if (uwsgi_buffer_append(ub, content_type, strlen(content_type))) goto error; if (uwsgi_buffer_append(ub, "\r\n", 2)) goto error; diff --git a/core/uwsgi.c b/core/uwsgi.c index 3d9a15e3f8..b236633996 100755 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -623,6 +623,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"stats", required_argument, 0, "enable the stats server on the specified address", uwsgi_opt_set_str, &uwsgi.stats, UWSGI_OPT_MASTER}, {"stats-server", required_argument, 0, "enable the stats server on the specified address", uwsgi_opt_set_str, &uwsgi.stats, UWSGI_OPT_MASTER}, {"stats-http", no_argument, 0, "prefix stats server json output with http headers", uwsgi_opt_true, &uwsgi.stats_http, UWSGI_OPT_MASTER}, + {"stats-prometheus-path", required_argument, 0, "set the path for prometheus metrics endpoint (default: /metrics)", uwsgi_opt_set_str, &uwsgi.stats_prometheus_path, UWSGI_OPT_MASTER}, {"stats-minified", no_argument, 0, "minify statistics json output", uwsgi_opt_true, &uwsgi.stats_minified, UWSGI_OPT_MASTER}, {"stats-min", no_argument, 0, "minify statistics json output", uwsgi_opt_true, &uwsgi.stats_minified, UWSGI_OPT_MASTER}, {"stats-push", required_argument, 0, "push the stats json to the specified destination", uwsgi_opt_add_string_list, &uwsgi.requested_stats_pushers, UWSGI_OPT_MASTER|UWSGI_OPT_METRICS}, diff --git a/plugins/corerouter/corerouter.c b/plugins/corerouter/corerouter.c index ea607351a7..db29e00f73 100644 --- a/plugins/corerouter/corerouter.c +++ b/plugins/corerouter/corerouter.c @@ -1058,7 +1058,12 @@ void corerouter_send_stats(struct uwsgi_corerouter *ucr) { } if (uwsgi.stats_http) { - if (uwsgi_send_http_stats(client_fd)) { + enum uwsgi_stats_format fmt; + if (uwsgi_stats_read_request(client_fd, &fmt)) { + close(client_fd); + return; + } + if (uwsgi_stats_send_http_header(client_fd, UWSGI_STATS_FORMAT_JSON)) { close(client_fd); return; } diff --git a/plugins/tuntap/tuntap.c b/plugins/tuntap/tuntap.c index dc8f821d60..6342b87591 100644 --- a/plugins/tuntap/tuntap.c +++ b/plugins/tuntap/tuntap.c @@ -463,7 +463,12 @@ void tuntaprouter_send_stats(struct uwsgi_tuntap_router *uttr) { } if (uwsgi.stats_http) { - if (uwsgi_send_http_stats(client_fd)) { + enum uwsgi_stats_format fmt; + if (uwsgi_stats_read_request(client_fd, &fmt)) { + close(client_fd); + return; + } + if (uwsgi_stats_send_http_header(client_fd, UWSGI_STATS_FORMAT_JSON)) { close(client_fd); return; } diff --git a/t/runner b/t/runner index b59966b5d4..d7be834083 100755 --- a/t/runner +++ b/t/runner @@ -25,6 +25,8 @@ UWSGI_PLUGINS = os.getenv("UWSGI_PLUGINS_TEST", "all").split(" ") UWSGI_ADDR = "127.0.0.1" UWSGI_PORT = 8000 UWSGI_HTTP = f"{UWSGI_ADDR}:{UWSGI_PORT}" +UWSGI_STATS_PORT = 9191 +UWSGI_STATS = f"{UWSGI_ADDR}:{UWSGI_STATS_PORT}" def plugins_available(plugins): @@ -46,15 +48,10 @@ class UwsgiTest(unittest.TestCase): text=True, ) - def uwsgi_ready(self): + def uwsgi_ready(self, port=UWSGI_PORT): try: s = socket.socket() - s.connect( - ( - UWSGI_ADDR, - UWSGI_PORT, - ) - ) + s.connect((UWSGI_ADDR, port)) except socket.error: return False else: @@ -62,16 +59,26 @@ class UwsgiTest(unittest.TestCase): finally: s.close() - def start_listen_server(self, args): - self.start_server(["--http-socket", UWSGI_HTTP] + args) - - # ensure server is ready - retries = 10 - while not self.uwsgi_ready() and retries > 0: + def wait_for_port(self, port, retries=10): + while not self.uwsgi_ready(port) and retries > 0: time.sleep(0.1) - retries = retries - 1 + retries -= 1 if retries == 0: - raise RuntimeError("uwsgi test server is not available") + raise RuntimeError(f"uwsgi not available on port {port}") + + def start_listen_server(self, args): + self.start_server(["--http-socket", UWSGI_HTTP] + args) + self.wait_for_port(UWSGI_PORT) + + def start_stats_server(self): + self.start_server([ + "--master", + "--need-app=0", + "--http-socket", UWSGI_HTTP, + "--stats", UWSGI_STATS, + "--stats-http", + ]) + self.wait_for_port(UWSGI_STATS_PORT) def tearDown(self): if hasattr(self._outcome, "errors"): @@ -254,5 +261,23 @@ class UwsgiTest(unittest.TestCase): self.assert_GET_body("/", "Hello") + def test_stats_json(self): + self.start_stats_server() + with requests.get(f"http://{UWSGI_STATS}/") as r: + self.assertEqual(r.status_code, 200) + self.assertIn("application/json", r.headers.get("Content-Type", "")) + data = r.json() + self.assertIn("version", data) + self.assertIn("workers", data) + + def test_stats_prometheus(self): + self.start_stats_server() + with requests.get(f"http://{UWSGI_STATS}/metrics") as r: + self.assertEqual(r.status_code, 200) + self.assertIn("text/plain", r.headers.get("Content-Type", "")) + self.assertIn("# TYPE uwsgi_workers gauge", r.text) + self.assertRegex(r.text, r"uwsgi_workers \d+") + + if __name__ == "__main__": unittest.main() diff --git a/uwsgi.h b/uwsgi.h index 748bc856a7..a8da88713d 100755 --- a/uwsgi.h +++ b/uwsgi.h @@ -2705,6 +2705,7 @@ struct uwsgi_server { char *stats; int stats_fd; int stats_http; + char *stats_prometheus_path; int stats_minified; struct uwsgi_string_list *requested_stats_pushers; struct uwsgi_stats_pusher *stats_pushers; @@ -4197,6 +4198,7 @@ void uwsgi_stats_pusher_loop(struct uwsgi_thread *); void uwsgi_stats_pusher_setup(void); void uwsgi_send_stats(int, struct uwsgi_stats *(*func) (void)); struct uwsgi_stats *uwsgi_master_generate_stats(void); +struct uwsgi_stats *uwsgi_master_generate_stats_prometheus(void); struct uwsgi_stats_pusher * uwsgi_register_stats_pusher(char *, void (*)(struct uwsgi_stats_pusher_instance *, time_t, char *, size_t)); struct uwsgi_stats *uwsgi_stats_new(size_t); @@ -4578,7 +4580,13 @@ void uwsgi_setup_thread_req(long, struct wsgi_request *); void uwsgi_loop_cores_run(void *(*)(void *)); int uwsgi_kvlist_parse(char *, size_t, char, int, ...); -int uwsgi_send_http_stats(int); +enum uwsgi_stats_format { + UWSGI_STATS_FORMAT_JSON = 0, + UWSGI_STATS_FORMAT_PROMETHEUS, +}; + +int uwsgi_stats_read_request(int, enum uwsgi_stats_format *); +int uwsgi_stats_send_http_header(int, enum uwsgi_stats_format); int uwsgi_plugin_modifier1(char *); diff --git a/uwsgiconfig.py b/uwsgiconfig.py index 3f64906589..19f9f60853 100644 --- a/uwsgiconfig.py +++ b/uwsgiconfig.py @@ -651,7 +651,7 @@ def __init__(self, filename, mute=False): self.gcc_list = [ 'core/utils', 'core/protocol', 'core/socket', 'core/logging', 'core/master', 'core/master_utils', 'core/emperor', 'core/notify', - 'core/mule', 'core/subscription', 'core/stats', 'core/sendfile', + 'core/mule', 'core/subscription', 'core/stats', 'core/stats_prometheus', 'core/sendfile', 'core/async', 'core/master_checks', 'core/fifo', 'core/offload', 'core/io', 'core/static', 'core/websockets', 'core/spooler', 'core/snmp', 'core/exceptions', 'core/config', 'core/setup_utils',