From 92be4078232cd0052c6110132f2b0bb3faec78bf Mon Sep 17 00:00:00 2001 From: jassuncao Date: Wed, 17 Jun 2026 15:55:05 +0100 Subject: [PATCH 1/2] NMS-19910: Add support for OpenMetrics v1 text format scraping Adds basic support for OpenMetrics v1 text format in Prometheus collector while retaining backwards compatibility with the legacy Prometheus v0.0.4 text format. Key changes: * Refactors `TextPrometheusMetricDataParser` to accept a strategy pattern via a new `TextFormatStrategy` interface. * Adds `PrometheusTextFormatStrategy` to handle Prometheus v0.0.4 metric format. * Adds `OpenMetricsFormatStrategy` and `OpenMetricsProcessor` to process OpenMetrics data. Supports the mandatory `_total` suffix in counters and the optional `_created` suffix. * Updates `PrometheusScraper` to dynamically inspect the HTTP response's `Content-Type` header and dispatch it to the correct processor. * Refactors the sample line parsing code into the SampleLineParser class. * Adds unit test to validate OpenMetrics parsing. Fixes based on PR review: * Attempts to make TextPrometheusMetricDataParser.parse() more robust. Instead of aborting when invalid metrics are found, it will skip over invalid lines until a valid metric family, or eof, are found. * Replace MetricType.UNTYPED with MetricType.UNKNOWN to match OpenMetrics UNKNOWN type. This type is used by default if no type is provided and will be treated as a gauge in PrometheusMetricsProcessor. * Make PrometheusScraper lenient with OpenMetrics Content-Type headers. * Corrects the software license in some files. * Adds tests for unknown, missing and invalid metric types in OpenMetricsTextV1Test * Adds a non-compliant OpenMetrics test in PrometheusCollectorIT --- features/prometheus-collector/pom.xml | 6 + .../PrometheusMetricsProcessor.java | 2 +- .../org/hawkular/agent/prometheus/Util.java | 26 ++ .../agent/prometheus/text/LineParser.java | 203 ++++++++++++++ .../text/OpenMetricsFormatStrategy.java | 186 +++++++++++++ .../prometheus/text/OpenMetricsProcessor.java | 48 ++++ .../text/PrometheusTextFormatStrategy.java | 151 +++++++++++ .../prometheus/text/SampleNameValidator.java | 60 +++++ .../prometheus/text/TextFormatStrategy.java | 33 +++ .../text/TextPrometheusMetricDataParser.java | 253 ++---------------- .../text/TextPrometheusMetricsProcessor.java | 2 +- .../prometheus/text/TextSampleProcessor.java | 37 +++ .../agent/prometheus/types/Counter.java | 12 + .../agent/prometheus/types/Gauge.java | 11 + .../agent/prometheus/types/Histogram.java | 7 + .../agent/prometheus/types/Metric.java | 34 +++ .../agent/prometheus/types/MetricFamily.java | 7 +- .../agent/prometheus/types/MetricType.java | 19 +- .../agent/prometheus/types/Summary.java | 11 + .../prometheus/PrometheusScraper.java | 42 ++- ...PrometheusDataCollectionConfigDaoJaxb.java | 1 - .../agent/prometheus/text/AssertUtils.java | 106 ++++++++ .../text/OpenMetricsTextV1Test.java | 160 +++++++++++ .../text/PrometheusTextV0_0_4Test.java | 108 ++++++++ .../prometheus/PrometheusCollectorIT.java | 88 +++++- .../src/test/resources/__files/open.metrics | 14 + .../__files/open_noncompliant.metrics | 4 + .../src/test/resources/mocklogger.properties | 2 + .../src/test/resources/openmetrics.txt | 43 +++ 29 files changed, 1436 insertions(+), 240 deletions(-) create mode 100644 features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/LineParser.java create mode 100644 features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/OpenMetricsFormatStrategy.java create mode 100644 features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/OpenMetricsProcessor.java create mode 100644 features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/PrometheusTextFormatStrategy.java create mode 100644 features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/SampleNameValidator.java create mode 100644 features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextFormatStrategy.java create mode 100644 features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextSampleProcessor.java create mode 100644 features/prometheus-collector/src/test/java/org/hawkular/agent/prometheus/text/AssertUtils.java create mode 100644 features/prometheus-collector/src/test/java/org/hawkular/agent/prometheus/text/OpenMetricsTextV1Test.java create mode 100644 features/prometheus-collector/src/test/java/org/hawkular/agent/prometheus/text/PrometheusTextV0_0_4Test.java create mode 100644 features/prometheus-collector/src/test/resources/__files/open.metrics create mode 100644 features/prometheus-collector/src/test/resources/__files/open_noncompliant.metrics create mode 100644 features/prometheus-collector/src/test/resources/mocklogger.properties create mode 100644 features/prometheus-collector/src/test/resources/openmetrics.txt diff --git a/features/prometheus-collector/pom.xml b/features/prometheus-collector/pom.xml index 50dfd9f04616..ad53ecd5deb0 100644 --- a/features/prometheus-collector/pom.xml +++ b/features/prometheus-collector/pom.xml @@ -142,5 +142,11 @@ 1.57 test + + javax.persistence + javax.persistence-api + 2.2 + test + diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/PrometheusMetricsProcessor.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/PrometheusMetricsProcessor.java index ea2ca4b1ae93..20ab67b33076 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/PrometheusMetricsProcessor.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/PrometheusMetricsProcessor.java @@ -96,7 +96,7 @@ public void walk() { ((org.hawkular.agent.prometheus.types.Histogram) metric), metricIndex); break; - case UNTYPED: + case UNKNOWN: // if we can't tell what it is, it's a gauge. log.debug("UNtyped metric: '{}'",metric); walker.walkGaugeMetric(convertedMetricFamily, (Gauge) metric, metricIndex); diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/Util.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/Util.java index 8603c9b9b56f..91eb74db5f23 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/Util.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/Util.java @@ -16,7 +16,18 @@ */ package org.hawkular.agent.prometheus; +import java.time.Instant; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class Util { + + private static final Logger LOGGER = LoggerFactory.getLogger(Util.class); + + private Util() { + // prevent instantiation + } public static double convertStringToDouble(String valueString) { double doubleValue; @@ -39,4 +50,19 @@ public static String convertDoubleToString(double value) { } return String.format("%f", value); } + + public static Instant tryConvertTimestamp(CharSequence timestamp) { + if (timestamp == null || timestamp.isEmpty()) { + return null; + } + try { + double timestampDouble = Double.parseDouble(timestamp.toString()); + long epochSecond = (long) timestampDouble; + return Instant.ofEpochSecond(epochSecond); + } + catch (NumberFormatException e) { + LOGGER.warn("Invalid timestamp: " + timestamp); + return null; + } + } } diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/LineParser.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/LineParser.java new file mode 100644 index 000000000000..263ec20f7745 --- /dev/null +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/LineParser.java @@ -0,0 +1,203 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.hawkular.agent.prometheus.text; + +import java.util.LinkedHashMap; +import java.util.Map; + +import org.hawkular.agent.prometheus.Util; + +class SampleLineParser { + + private enum ParseState { + NAME, + ENDOFNAME, + STARTOFLABELNAME, + LABELNAME, + LABELVALUEEQUALS, + LABELVALUEQUOTE, + LABELVALUE, + LABELVALUESLASH, + NEXTLABEL, + ENDOFLABELS, + VALUE, + TIMESTAMP + } + + private final StringBuilder name = new StringBuilder(); + private final StringBuilder labelname = new StringBuilder(); + private final StringBuilder labelvalue = new StringBuilder(); + private final StringBuilder value = new StringBuilder(); + private final StringBuilder timestamp = new StringBuilder(); + private final Map labels = new LinkedHashMap<>(); + private ParseState state; + + public void reset() { + name.setLength(0); + labelname.setLength(0); + labelvalue.setLength(0); + value.setLength(0); + timestamp.setLength(0); + labels.clear(); + state = ParseState.NAME; + } + + public TextSample parse(String line) { + reset(); + + for (int c = 0; c < line.length(); c++) { + char charAt = line.charAt(c); + + switch (state) { + case NAME: + if (charAt == '{') { + state = ParseState.STARTOFLABELNAME; + } else if (charAt == ' ' || charAt == '\t') { + state = ParseState.ENDOFNAME; + } else { + name.append(charAt); + } + break; + case ENDOFNAME: + if (charAt == ' ' || charAt == '\t') { + // do nothing + } else if (charAt == '{') { + state = ParseState.STARTOFLABELNAME; + } else { + value.append(charAt); + state = ParseState.VALUE; + } + break; + case STARTOFLABELNAME: + if (charAt == ' ' || charAt == '\t') { + // do nothing + } else if (charAt == '}') { + state = ParseState.ENDOFLABELS; + } else { + labelname.append(charAt); + state = ParseState.LABELNAME; + } + break; + case LABELNAME: + if (charAt == '=') { + state = ParseState.LABELVALUEQUOTE; + } else if (charAt == '}') { + state = ParseState.ENDOFLABELS; + } else if (charAt == ' ' || charAt == '\t') { + state = ParseState.LABELVALUEEQUALS; + } else { + labelname.append(charAt); + } + break; + case LABELVALUEEQUALS: + if (charAt == '=') { + state = ParseState.LABELVALUEQUOTE; + } else if (charAt == ' ' || charAt == '\t') { + // do nothing + } else { + throw new IllegalStateException("Invalid line: " + line); + } + break; + case LABELVALUEQUOTE: + if (charAt == '"') { + state = ParseState.LABELVALUE; + } else if (charAt == ' ' || charAt == '\t') { + // do nothing + } else { + throw new IllegalStateException("Invalid line: " + line); + } + break; + case LABELVALUE: + if (charAt == '\\') { + state = ParseState.LABELVALUESLASH; + } else if (charAt == '"') { + labels.put(labelname.toString(), labelvalue.toString()); + labelname.setLength(0); + labelvalue.setLength(0); + state = ParseState.NEXTLABEL; + } else { + labelvalue.append(charAt); + } + break; + case LABELVALUESLASH: + state = ParseState.LABELVALUE; + if (charAt == '\\') { + labelvalue.append('\\'); + } else if (charAt == 'n') { + labelvalue.append('\n'); + } else if (charAt == '"') { + labelvalue.append('"'); + } else { + labelvalue.append('\\').append(charAt); + } + break; + case NEXTLABEL: + if (charAt == ',') { + state = ParseState.LABELNAME; + } else if (charAt == '}') { + state = ParseState.ENDOFLABELS; + } else if (charAt == ' ' || charAt == '\t') { + // do nothing + } else { + throw new IllegalStateException("Invalid line: " + line); + } + break; + case ENDOFLABELS: + if (charAt == ' ' || charAt == '\t') { + // do nothing + } else { + value.append(charAt); + state = ParseState.VALUE; + } + break; + case VALUE: + if (charAt == ' ' || charAt == '\t') { + state = ParseState.TIMESTAMP; + } else { + value.append(charAt); + } + break; + case TIMESTAMP: + if (charAt == ' ' || charAt == '\t') { + return buildSample(line); + } + else { + timestamp.append(charAt); + } + break; + } + } + + return buildSample(line); + } + + private TextSample buildSample(String line) { + return new TextSample.Builder() + .setLine(line) + .setName(name.toString()) + .setValue(value.toString()) + .setTimestamp(Util.tryConvertTimestamp(timestamp)) + .addLabels(new LinkedHashMap<>(labels)) + .build(); + } + +} \ No newline at end of file diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/OpenMetricsFormatStrategy.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/OpenMetricsFormatStrategy.java new file mode 100644 index 000000000000..715bcbc9d491 --- /dev/null +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/OpenMetricsFormatStrategy.java @@ -0,0 +1,186 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.hawkular.agent.prometheus.text; + +import java.util.Map; + +import org.hawkular.agent.prometheus.Util; +import org.hawkular.agent.prometheus.types.Counter; +import org.hawkular.agent.prometheus.types.Gauge; +import org.hawkular.agent.prometheus.types.Histogram; +import org.hawkular.agent.prometheus.types.Metric; +import org.hawkular.agent.prometheus.types.MetricType; +import org.hawkular.agent.prometheus.types.Summary; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + * Strategy for processing OpenMetrics v1 text data. + *

+ * The strategy is very similar to the Prometheus one but counters have a mandatory _total suffix and there is supported for the "created" data sample. + *

+ * + */ +enum OpenMetricsFormatStrategy implements TextFormatStrategy { + INSTANCE; + + private static final Logger LOGGER = LoggerFactory.getLogger(OpenMetricsFormatStrategy.class); + + @Override + public TextSampleProcessor getSampleProcessor(MetricType type) { + + switch (type) { + case COUNTER: + return OpenMetricsFormatStrategy::handleCounterSample; + case GAUGE: + //treat UNKNOWN as gauge + case UNKNOWN: + return OpenMetricsFormatStrategy::handleGaugeSample; + case SUMMARY: + return OpenMetricsFormatStrategy::handleSummarySample; + case HISTOGRAM: + return OpenMetricsFormatStrategy::handleHistogramSample; + default: + throw new IllegalArgumentException("Unknown metric type: " + type); + } + } + + @Override + public SampleNameValidator getNameValidator(MetricType type, String familyName) { + + switch (type) { + case COUNTER: + return SampleNameValidator.matchingNameWithSuffixes(familyName, "_total", "_created"); + case GAUGE: + case UNKNOWN: + return SampleNameValidator.matchingNameWithSuffixes(familyName, "", "_created"); + case SUMMARY: + return SampleNameValidator.matchingNameWithSuffixes(familyName, "", "_count", "_sum", "_created"); + case HISTOGRAM: + return SampleNameValidator.matchingNameWithSuffixes(familyName, "_count", "_sum", "_bucket", "_created"); + default: + throw new IllegalArgumentException("Unknown metric type: " + type); + } + } + + + private static void handleCounterSample(Map, Metric.Builder> builders, String familyName, TextSample textSample) { + Counter.Builder cBuilder = (Counter.Builder) builders.get(textSample.getLabels()); + if (cBuilder == null) { + cBuilder = Counter.builder() + .setName(familyName) + .addLabels(textSample.getLabels()); + builders.put(textSample.getLabels(), cBuilder); + } + if (textSample.getName().endsWith("_created")) { + cBuilder.setCreated(Util.tryConvertTimestamp(textSample.getValue())); + } + else { + cBuilder + .setValue(Util.convertStringToDouble(textSample.getValue())) + .setTimestamp(textSample.getTimestamp()); + } + } + + private static void handleGaugeSample(Map, Metric.Builder> builders, String familyName, TextSample textSample) { + Gauge.Builder gBuilder = (Gauge.Builder) builders.get(textSample.getLabels()); + if (gBuilder == null) { + gBuilder = Gauge.builder() + .setName(familyName) + .addLabels(textSample.getLabels()); + builders.put(textSample.getLabels(), gBuilder); + } + if (textSample.getName().endsWith("_created")) { + gBuilder.setCreated(Util.tryConvertTimestamp(textSample.getValue())); + } + else { + gBuilder.setValue(Util.convertStringToDouble(textSample.getValue())) + .setTimestamp(textSample.getTimestamp()); + } + } + + private static void handleSummarySample(Map, Metric.Builder> builders, String familyName, TextSample textSample) { + // First we need to remove any existing quantile label since it isn't a "real" label. + // This is to ensure our lookup uses all but only "real" labels. + String quantileValue = textSample.getLabels().remove("quantile"); // may be null + + Summary.Builder sBuilder = (Summary.Builder) builders.get(textSample.getLabels()); + if (sBuilder == null) { + sBuilder = new Summary.Builder(); + sBuilder.setName(familyName); + sBuilder.addLabels(textSample.getLabels()); + builders.put(textSample.getLabels(), sBuilder); + } + if (textSample.getName().endsWith("_count")) { + sBuilder.setSampleCount((long)Util.convertStringToDouble(textSample.getValue())); + } else if (textSample.getName().endsWith("_sum")) { + sBuilder.setSampleSum(Util.convertStringToDouble(textSample.getValue())); + } + else if (textSample.getName().endsWith("_created")) { + sBuilder.setCreated(Util.tryConvertTimestamp(textSample.getValue())); + } + else { + // This must be a quantile sample + if (quantileValue == null) { + LOGGER.debug("Summary quantile sample is missing the 'quantile' label: {}", + textSample.getLine()); + } + sBuilder.addQuantile(Util.convertStringToDouble(quantileValue), + Util.convertStringToDouble(textSample.getValue())); + } + } + + private static void handleHistogramSample(Map, Metric.Builder> builders, String familyName, TextSample textSample) { + // Get the builder that we are using to build up the current metric. Remember we need to + // get the builder for this specific metric identified with a unique set of labels. + + // First we need to remove any existing le label since it isn't a "real" label. + // This is to ensure our lookup uses all but only "real" labels. + String bucket = textSample.getLabels().remove("le"); // may be null + + Histogram.Builder hBuilder = (Histogram.Builder) builders.get(textSample.getLabels()); + if (hBuilder == null) { + hBuilder = new Histogram.Builder(); + hBuilder.setName(familyName); + hBuilder.addLabels(textSample.getLabels()); + builders.put(textSample.getLabels(), hBuilder); + } + if (textSample.getName().endsWith("_count")) { + hBuilder.setSampleCount((long)Util.convertStringToDouble(textSample.getValue())); + } else if (textSample.getName().endsWith("_sum")) { + hBuilder.setSampleSum(Util.convertStringToDouble(textSample.getValue())); + } + else if (textSample.getName().endsWith("_created")) { + hBuilder.setCreated(Util.tryConvertTimestamp(textSample.getValue())); + } + else { + // This must be a bucket sample + if (bucket == null) { + throw new IllegalArgumentException("Histogram bucket sample is missing the 'le' label"); + } + hBuilder.addBucket(Util.convertStringToDouble(bucket), + (long)Util.convertStringToDouble(textSample.getValue())); + } + } + +} diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/OpenMetricsProcessor.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/OpenMetricsProcessor.java new file mode 100644 index 000000000000..00ede0ba434c --- /dev/null +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/OpenMetricsProcessor.java @@ -0,0 +1,48 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.hawkular.agent.prometheus.text; + +import java.io.InputStream; + +import org.hawkular.agent.prometheus.PrometheusMetricsProcessor; +import org.hawkular.agent.prometheus.types.MetricFamily; +import org.hawkular.agent.prometheus.walkers.PrometheusMetricsWalker; + +/** + * This will iterate over a list of Prometheus metrics that are given in OpenMetrics text data. + */ +public class OpenMetricsProcessor extends PrometheusMetricsProcessor { + public OpenMetricsProcessor(InputStream inputStream, PrometheusMetricsWalker theWalker) { + super(inputStream, theWalker); + } + + @Override + public TextPrometheusMetricDataParser createPrometheusMetricDataParser() { + return new TextPrometheusMetricDataParser(getInputStream(), OpenMetricsFormatStrategy.INSTANCE); + } + + @Override + protected MetricFamily convert(MetricFamily metricFamily) { + return metricFamily; // no conversion necessary - our text parser already uses the common api + } + +} diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/PrometheusTextFormatStrategy.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/PrometheusTextFormatStrategy.java new file mode 100644 index 000000000000..0555ab5d9bec --- /dev/null +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/PrometheusTextFormatStrategy.java @@ -0,0 +1,151 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.hawkular.agent.prometheus.text; + +import java.util.Map; + +import org.hawkular.agent.prometheus.Util; +import org.hawkular.agent.prometheus.types.Counter; +import org.hawkular.agent.prometheus.types.Gauge; +import org.hawkular.agent.prometheus.types.Histogram; +import org.hawkular.agent.prometheus.types.Metric; +import org.hawkular.agent.prometheus.types.MetricType; +import org.hawkular.agent.prometheus.types.Summary; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +enum PrometheusTextFormatStrategy implements TextFormatStrategy { + + INSTANCE; + + private static final Logger LOGGER = LoggerFactory.getLogger(PrometheusTextFormatStrategy.class); + + @Override + public TextSampleProcessor getSampleProcessor(MetricType type) { + + switch (type) { + case COUNTER: + return PrometheusTextFormatStrategy::handleCounterSample; + case GAUGE: + // treat UNKNOWN as gauge + case UNKNOWN: + return PrometheusTextFormatStrategy::handleGaugeSample; + case HISTOGRAM: + return PrometheusTextFormatStrategy::handleHistogramSample; + case SUMMARY: + return PrometheusTextFormatStrategy::handleSummarySample; + default: + throw new IllegalArgumentException("Unknown metric type: " + type); + } + } + + @Override + public SampleNameValidator getNameValidator(MetricType type, String familyName) { + switch (type) { + case COUNTER: + case GAUGE: + case UNKNOWN: + return SampleNameValidator.exactName(familyName); + case HISTOGRAM: + return SampleNameValidator.matchingNameWithSuffixes(familyName, "", "_count", "_sum", "_bucket"); + case SUMMARY: + return SampleNameValidator.matchingNameWithSuffixes(familyName, "", "_count", "_sum"); + default: + throw new IllegalArgumentException("Unknown metric type: " + type); + } + } + + static void handleCounterSample(Map, Metric.Builder> builders, String familyName, TextSample textSample) { + builders.put(textSample.getLabels(), + new Counter.Builder().setName(familyName) + .setValue(Util.convertStringToDouble(textSample.getValue())) + .addLabels(textSample.getLabels())); + } + + static void handleGaugeSample(Map, Metric.Builder> builders, String familyName, TextSample textSample) { + builders.put(textSample.getLabels(), + new Gauge.Builder().setName(familyName) + .setValue(Util.convertStringToDouble(textSample.getValue())) + .addLabels(textSample.getLabels())); + } + + static void handleSummarySample(Map, Metric.Builder> builders, String familyName, TextSample textSample) { + // Get the builder that we are using to build up the current metric. Remember we need to + // get the builder for this specific metric identified with a unique set of labels. + + // First we need to remove any existing quantile label since it isn't a "real" label. + // This is to ensure our lookup uses all but only "real" labels. + String quantileValue = textSample.getLabels().remove("quantile"); // may be null + + Summary.Builder sBuilder = (Summary.Builder) builders.get(textSample.getLabels()); + if (sBuilder == null) { + sBuilder = new Summary.Builder(); + sBuilder.setName(familyName); + sBuilder.addLabels(textSample.getLabels()); + builders.put(textSample.getLabels(), sBuilder); + } + if (textSample.getName().endsWith("_count")) { + sBuilder.setSampleCount((long)Util.convertStringToDouble(textSample.getValue())); + } else if (textSample.getName().endsWith("_sum")) { + sBuilder.setSampleSum(Util.convertStringToDouble(textSample.getValue())); + } else { + // This must be a quantile sample + if (quantileValue == null) { + LOGGER.debug("Summary quantile sample is missing the 'quantile' label: {}", + textSample.getLine()); + } + sBuilder.addQuantile(Util.convertStringToDouble(quantileValue), + Util.convertStringToDouble(textSample.getValue())); + } + } + + static void handleHistogramSample(Map, Metric.Builder> builders, String familyName, TextSample textSample) { + // Get the builder that we are using to build up the current metric. Remember we need to + // get the builder for this specific metric identified with a unique set of labels. + + // First we need to remove any existing le label since it isn't a "real" label. + // This is to ensure our lookup uses all but only "real" labels. + String bucket = textSample.getLabels().remove("le"); // may be null + + Histogram.Builder hBuilder = (Histogram.Builder) builders.get(textSample.getLabels()); + if (hBuilder == null) { + hBuilder = new Histogram.Builder(); + hBuilder.setName(familyName); + hBuilder.addLabels(textSample.getLabels()); + builders.put(textSample.getLabels(), hBuilder); + } + if (textSample.getName().endsWith("_count")) { + hBuilder.setSampleCount((long)Util.convertStringToDouble(textSample.getValue())); + } else if (textSample.getName().endsWith("_sum")) { + hBuilder.setSampleSum(Util.convertStringToDouble(textSample.getValue())); + } else { + // This must be a bucket sample + if (bucket == null) { + throw new IllegalArgumentException("Histogram bucket sample is missing the 'le' label"); + } + hBuilder.addBucket(Util.convertStringToDouble(bucket), + (long)Util.convertStringToDouble(textSample.getValue())); + } + } + + +} diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/SampleNameValidator.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/SampleNameValidator.java new file mode 100644 index 000000000000..f893dbbf3764 --- /dev/null +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/SampleNameValidator.java @@ -0,0 +1,60 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.hawkular.agent.prometheus.text; + +interface SampleNameValidator { + boolean isValid(String sampleName); + + public static SampleNameValidator rejectAll() { + return sampleName -> false; + } + + /** + * Returns a validator that only accepts sample names that exactly match the given family name. + * @param familyName + * @return + */ + public static SampleNameValidator exactName(String familyName) { + return sampleName -> familyName.equals(sampleName); + } + + /** + * Returns a validator that only accepts sample names that start with the given family name and end with one of the given suffixes. + * @param familyName + * @param suffixes + * @return + */ + public static SampleNameValidator matchingNameWithSuffixes(String familyName, String... suffixes) { + return sampleName -> { + if(!sampleName.startsWith(familyName)) { + return false; + } + String suffix = sampleName.substring(familyName.length()); + for (String s : suffixes) { + if (s.equals(suffix)) { + return true; + } + } + return false; + }; + } +} diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextFormatStrategy.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextFormatStrategy.java new file mode 100644 index 000000000000..b02dd40f858d --- /dev/null +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextFormatStrategy.java @@ -0,0 +1,33 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.hawkular.agent.prometheus.text; + +import org.hawkular.agent.prometheus.types.MetricType; + +/** + * Strategy for processing text samples based on the metric type. + * + */ +interface TextFormatStrategy { + TextSampleProcessor getSampleProcessor(MetricType type); + SampleNameValidator getNameValidator(MetricType type, String familyName); +} diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextPrometheusMetricDataParser.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextPrometheusMetricDataParser.java index a03f5548b44a..3e53eaac8a21 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextPrometheusMetricDataParser.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextPrometheusMetricDataParser.java @@ -24,15 +24,11 @@ import java.util.List; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.hawkular.agent.prometheus.PrometheusMetricDataParser; -import org.hawkular.agent.prometheus.Util; -import org.hawkular.agent.prometheus.types.Counter; -import org.hawkular.agent.prometheus.types.Gauge; -import org.hawkular.agent.prometheus.types.Histogram; import org.hawkular.agent.prometheus.types.Metric; import org.hawkular.agent.prometheus.types.MetricFamily; import org.hawkular.agent.prometheus.types.MetricType; -import org.hawkular.agent.prometheus.types.Summary; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,6 +39,9 @@ public class TextPrometheusMetricDataParser extends PrometheusMetricDataParser allowedNames = new ArrayList<>(); public List textSamples = new ArrayList<>(); + public SampleNameValidator sampleNameValidator = SampleNameValidator.rejectAll(); // starts a fresh metric family public void clear() { name = ""; help = ""; type = null; - allowedNames.clear(); + sampleNameValidator = SampleNameValidator.rejectAll(); textSamples.clear(); } @@ -92,82 +92,12 @@ public void finishMetricFamily() { // For histogram metrics, we need to combine all bucket samples, sum, and count. Map, Metric.Builder> builders = new LinkedHashMap<>(); + + TextSampleProcessor textSampleProcessor = strategy.getSampleProcessor(type); for (TextSample textSample : textSamples) { try { - switch (type) { - case COUNTER: - builders.put(textSample.getLabels(), - new Counter.Builder().setName(name) - .setValue(Util.convertStringToDouble(textSample.getValue())) - .addLabels(textSample.getLabels())); - break; - case UNTYPED: - //treat UNTYPED as gauge - case GAUGE: - builders.put(textSample.getLabels(), - new Gauge.Builder().setName(name) - .setValue(Util.convertStringToDouble(textSample.getValue())) - .addLabels(textSample.getLabels())); - break; - case SUMMARY: - // Get the builder that we are using to build up the current metric. Remember we need to - // get the builder for this specific metric identified with a unique set of labels. - - // First we need to remove any existing quantile label since it isn't a "real" label. - // This is to ensure our lookup uses all but only "real" labels. - String quantileValue = textSample.getLabels().remove("quantile"); // may be null - - Summary.Builder sBuilder = (Summary.Builder) builders.get(textSample.getLabels()); - if (sBuilder == null) { - sBuilder = new Summary.Builder(); - builders.put(textSample.getLabels(), sBuilder); - } - sBuilder.setName(name); - sBuilder.addLabels(textSample.getLabels()); - if (textSample.getName().endsWith("_count")) { - sBuilder.setSampleCount((long)Util.convertStringToDouble(textSample.getValue())); - } else if (textSample.getName().endsWith("_sum")) { - sBuilder.setSampleSum(Util.convertStringToDouble(textSample.getValue())); - } else { - // This must be a quantile sample - if (quantileValue == null) { - log.debug("Summary quantile sample is missing the 'quantile' label: {}", - textSample.getLine()); - } - sBuilder.addQuantile(Util.convertStringToDouble(quantileValue), - Util.convertStringToDouble(textSample.getValue())); - } - break; - case HISTOGRAM: - // Get the builder that we are using to build up the current metric. Remember we need to - // get the builder for this specific metric identified with a unique set of labels. - - // First we need to remove any existing le label since it isn't a "real" label. - // This is to ensure our lookup uses all but only "real" labels. - String bucket = textSample.getLabels().remove("le"); // may be null - - Histogram.Builder hBuilder = (Histogram.Builder) builders.get(textSample.getLabels()); - if (hBuilder == null) { - hBuilder = new Histogram.Builder(); - builders.put(textSample.getLabels(), hBuilder); - } - hBuilder.setName(name); - hBuilder.addLabels(textSample.getLabels()); - if (textSample.getName().endsWith("_count")) { - hBuilder.setSampleCount((long)Util.convertStringToDouble(textSample.getValue())); - } else if (textSample.getName().endsWith("_sum")) { - hBuilder.setSampleSum(Util.convertStringToDouble(textSample.getValue())); - } else { - // This must be a bucket sample - if (bucket == null) { - throw new Exception("Histogram bucket sample is missing the 'le' label"); - } - hBuilder.addBucket(Util.convertStringToDouble(bucket), - (long)Util.convertStringToDouble(textSample.getValue())); - } - break; - } + textSampleProcessor.process(builders, name, textSample); } catch (Exception e) { log.debug("Error processing sample. This metric sample will be ignored: {}", textSample.getLine(), e); @@ -233,8 +163,8 @@ public MetricFamily parse() throws IOException { // start anew context.clear(); context.name = parts[2]; - context.type = MetricType.GAUGE; // default in case we don't get a TYPE - context.allowedNames.add(parts[2]); + context.type = MetricType.UNKNOWN; // default in case we don't get a TYPE + context.sampleNameValidator = strategy.getNameValidator(context.type, context.name); } if (parts.length == 4) { @@ -253,36 +183,22 @@ public MetricFamily parse() throws IOException { context.clear(); context.name = parts[2]; } - context.type = MetricType.valueOf(parts[3].toUpperCase()); - context.allowedNames.clear(); - switch (context.type) { - case COUNTER: - context.allowedNames.add(context.name); - break; - case GAUGE: - context.allowedNames.add(context.name); - break; - case UNTYPED: - context.allowedNames.add(context.name); - break; - case SUMMARY: - context.allowedNames.add(context.name + "_count"); - context.allowedNames.add(context.name + "_sum"); - context.allowedNames.add(context.name); - break; - case HISTOGRAM: - context.allowedNames.add(context.name + "_count"); - context.allowedNames.add(context.name + "_sum"); - context.allowedNames.add(context.name + "_bucket"); - break; + String type = parts.length < 4 ? "" : parts[3]; + context.type = StringUtils.isEmpty(type) ? MetricType.UNKNOWN : MetricType.tryParse(type); + if(context.type != null) { + context.sampleNameValidator = strategy.getNameValidator(context.type, context.name); } + else { + // we don't recognize this type, so we will ignore all samples for this family + context.clear(); + } } else { // ignore other tokens - probably a comment } } else { // parse the sample line that contains a single metric (or part of a metric as in summary/histo) - TextSample sample = parseSampleLine(line); - if (!context.allowedNames.contains(sample.getName())) { + TextSample sample = sampleLineParser.parse(line); + if (!context.sampleNameValidator.isValid(sample.getName())) { if (!context.name.isEmpty()) { // break and we'll finish the metric family we previously were building up this.lastLineReadFromStream = line; @@ -304,7 +220,7 @@ public MetricFamily parse() throws IOException { line = readLine(getInputStream()); } - if (!context.name.isEmpty()) { + if (!context.name.isEmpty() && context.type != null) { // finish the metric family we previously were building up context.finishMetricFamily(); } @@ -312,127 +228,6 @@ public MetricFamily parse() throws IOException { return context.finishedMetricFamily; } - private TextSample parseSampleLine(String line) { - // algorithm from parser.py - StringBuilder name = new StringBuilder(); - StringBuilder labelname = new StringBuilder(); - StringBuilder labelvalue = new StringBuilder(); - StringBuilder value = new StringBuilder(); - Map labels = new LinkedHashMap<>(); - - String state = "name"; - - for (int c = 0; c < line.length(); c++) { - char charAt = line.charAt(c); - if (state.equals("name")) { - if (charAt == '{') { - state = "startoflabelname"; - } else if (charAt == ' ' || charAt == '\t') { - state = "endofname"; - } else { - name.append(charAt); - } - } else if (state.equals("endofname")) { - if (charAt == ' ' || charAt == '\t') { - // do nothing - } else if (charAt == '{') { - state = "startoflabelname"; - } else { - value.append(charAt); - state = "value"; - } - } else if (state.equals("startoflabelname")) { - if (charAt == ' ' || charAt == '\t') { - // do nothing - } else if (charAt == '}') { - state = "endoflabels"; - } else { - labelname.append(charAt); - state = "labelname"; - } - } else if (state.equals("labelname")) { - if (charAt == '=') { - state = "labelvaluequote"; - } else if (charAt == '}') { - state = "endoflabels"; - } else if (charAt == ' ' || charAt == '\t') { - state = "labelvalueequals"; - } else { - labelname.append(charAt); - } - } else if (state.equals("labelvalueequals")) { - if (charAt == '=') { - state = "labelvaluequote"; - } else if (charAt == ' ' || charAt == '\t') { - // do nothing - } else { - throw new IllegalStateException("Invalid line: " + line); - } - } else if (state.equals("labelvaluequote")) { - if (charAt == '"') { - state = "labelvalue"; - } else if (charAt == ' ' || charAt == '\t') { - // do nothing - } else { - throw new IllegalStateException("Invalid line: " + line); - } - } else if (state.equals("labelvalue")) { - if (charAt == '\\') { - state = "labelvalueslash"; - } else if (charAt == '"') { - labels.put(labelname.toString(), labelvalue.toString()); - labelname.setLength(0); - labelvalue.setLength(0); - state = "nextlabel"; - } else { - labelvalue.append(charAt); - } - } else if (state.equals("labelvalueslash")) { - state = "labelvalue"; - if (charAt == '\\') { - labelvalue.append('\\'); - } else if (charAt == 'n') { - labelvalue.append('\n'); - } else if (charAt == '"') { - labelvalue.append('"'); - } else { - labelvalue.append('\\').append(charAt); - } - } else if (state.equals("nextlabel")) { - if (charAt == ',') { - state = "labelname"; - } else if (charAt == '}') { - state = "endoflabels"; - } else if (charAt == ' ' || charAt == '\t') { - // do nothing - } else { - throw new IllegalStateException("Invalid line: " + line); - } - } else if (state.equals("endoflabels")) { - if (charAt == ' ' || charAt == '\t') { - // do nothing - } else { - value.append(charAt); - state = "value"; - } - } else if (state.equals("value")) { - if (charAt == ' ' || charAt == '\t') { - break; // timestamps are NOT supported - ignoring - } else { - value.append(charAt); - } - } - } - - TextSample sample = new TextSample.Builder() - .setLine(line) - .setName(name.toString()) - .setValue(value.toString()) - .addLabels(labels).build(); - - return sample; - } - private String unescapeHelp(String text) { // algorithm from parser.py if (text == null || !text.contains("\\")) { diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextPrometheusMetricsProcessor.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextPrometheusMetricsProcessor.java index cb5777d21961..4063026b47a7 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextPrometheusMetricsProcessor.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextPrometheusMetricsProcessor.java @@ -32,7 +32,7 @@ public TextPrometheusMetricsProcessor(InputStream inputStream, PrometheusMetrics @Override public TextPrometheusMetricDataParser createPrometheusMetricDataParser() { - return new TextPrometheusMetricDataParser(getInputStream()); + return new TextPrometheusMetricDataParser(getInputStream(), PrometheusTextFormatStrategy.INSTANCE); } @Override diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextSampleProcessor.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextSampleProcessor.java new file mode 100644 index 000000000000..41d0edf99de1 --- /dev/null +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextSampleProcessor.java @@ -0,0 +1,37 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.hawkular.agent.prometheus.text; + +import java.util.Map; + +import org.hawkular.agent.prometheus.types.Metric; + +@FunctionalInterface +interface TextSampleProcessor { + /** + * Processes a Prometheus text sample and adds it to the given builders map. + * @param builders a map of builders for each metric family, keyed by a map of labels to the builder + * @param familyName the name of the metric family + * @param textSample the text sample to process + */ + void process(Map, Metric.Builder> builders, String familyName, TextSample textSample); +} diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Counter.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Counter.java index 732f911539f6..b84b9a557eb8 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Counter.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Counter.java @@ -42,9 +42,21 @@ public Counter(Builder builder) { public double getValue() { return value; } + + @Override + public String toString() { + + return "Counter [name=" + getName() + ", labels" + getLabels() + ", timestamp=" + getTimestamp() + ", created=" + getCreated() + + ", value=" + value + "]"; + } @Override public void visit(MetricVisitor visitor) { visitor.visitCounter(this); } + + public static Builder builder() { + + return new Builder(); + } } diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Gauge.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Gauge.java index b6fae94b35f6..e040000b7835 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Gauge.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Gauge.java @@ -43,8 +43,19 @@ public double getValue() { return value; } + @Override + public String toString() { + + return "Gauge [name=" + getName() + ", labels" + getLabels() + ", timestamp=" + getTimestamp() + ", created=" + getCreated() + + ", value=" + value + "]"; + } + @Override public void visit(MetricVisitor visitor) { visitor.visitGauge(this); } + + public static Builder builder() { + return new Builder(); + } } diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Histogram.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Histogram.java index 70eda6763628..cd1832a5c3e7 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Histogram.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Histogram.java @@ -110,6 +110,13 @@ public List getBuckets() { } return buckets; } + + @Override + public String toString() { + + return "Histogram [name=" + getName() + ", labels" + getLabels() + ", timestamp=" + getTimestamp() + ", created=" + getCreated() + + ", sampleCount=" + sampleCount + ", sampleSum=" + sampleSum + "]"; + } @Override public void visit(MetricVisitor visitor) { diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Metric.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Metric.java index 7f236313bdfc..d8fc5b1349d4 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Metric.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Metric.java @@ -16,6 +16,7 @@ */ package org.hawkular.agent.prometheus.types; +import java.time.Instant; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; @@ -28,6 +29,8 @@ public abstract class Metric { public abstract static class Builder> { private String name; private Map labels; + private Instant timestamp; + private Instant created; @SuppressWarnings("unchecked") public B setName(String name) { @@ -52,12 +55,26 @@ public B addLabels(Map map) { labels.putAll(map); return (B) this; } + + @SuppressWarnings("unchecked") + public B setTimestamp(Instant timestamp) { + this.timestamp = timestamp; + return (B) this; + } + + @SuppressWarnings("unchecked") + public B setCreated(Instant created) { + this.created = created; + return (B) this; + } public abstract T build(); } private final String name; private final Map labels; + private final Instant timestamp; + private final Instant created; protected Metric(Builder builder) { if (builder.name == null) { @@ -66,6 +83,8 @@ protected Metric(Builder builder) { this.name = builder.name; this.labels = builder.labels; + this.timestamp = builder.timestamp; + this.created = builder.created; } public String getName() { @@ -79,5 +98,20 @@ public Map getLabels() { return labels; } + public Instant getTimestamp() { + return timestamp; + } + + public Instant getCreated() { + + return created; + } + + @Override + public String toString() { + + return "Metric [name=" + name + ", labels=" + labels + ", timestamp=" + timestamp + ", created=" + created + "]"; + } + public abstract void visit(MetricVisitor visitor); } diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/MetricFamily.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/MetricFamily.java index a38b6e47e0a4..62147b37715a 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/MetricFamily.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/MetricFamily.java @@ -87,7 +87,7 @@ protected MetricFamily(Builder builder) { case HISTOGRAM: expectedMetricClassType = Histogram.class; break; - case UNTYPED: + case UNKNOWN: //treat untyped metrics as gauge expectedMetricClassType = Gauge.class; break; @@ -132,4 +132,9 @@ public List getMetrics() { } return metrics; } + + @Override + public String toString() { + return "MetricFamily [name=" + name + ", help=" + help + ", type=" + type + ", metrics=" + metrics + "]"; + } } diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/MetricType.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/MetricType.java index 72e267d915f3..41a6a407d45a 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/MetricType.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/MetricType.java @@ -18,5 +18,22 @@ package org.hawkular.agent.prometheus.types; public enum MetricType { - COUNTER, GAUGE, SUMMARY, HISTOGRAM, UNTYPED + COUNTER, GAUGE, SUMMARY, HISTOGRAM, UNKNOWN; + + /** + * Try to parse the given type string into a MetricType enum value. If the type is not recognized, it will return null. + * @param value the type string to parse. + * @return the MetricType enum value or null if not recognized + */ + public static MetricType tryParse(String value) { + if (value == null) { + return null; + } + try { + return MetricType.valueOf(value.toUpperCase()); + } + catch (IllegalArgumentException e) { + return null; + } + } } diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Summary.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Summary.java index 45b135823611..b902e41cf235 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Summary.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Summary.java @@ -111,9 +111,20 @@ public List getQuantiles() { } return quantiles; } + + @Override + public String toString() { + + return "Summary [name=" + getName() + ", labels" + getLabels() + ", timestamp=" + getTimestamp() + ", created=" + getCreated() + + ", sampleCount=" + sampleCount + ", sampleSum=" + sampleSum + "]"; + } @Override public void visit(MetricVisitor visitor) { visitor.visitSummary(this); } + + public static Builder builder() { + return new Builder(); + } } diff --git a/features/prometheus-collector/src/main/java/org/opennms/netmgt/collectd/prometheus/PrometheusScraper.java b/features/prometheus-collector/src/main/java/org/opennms/netmgt/collectd/prometheus/PrometheusScraper.java index 6d88d35d1b9a..275a0412db5d 100644 --- a/features/prometheus-collector/src/main/java/org/opennms/netmgt/collectd/prometheus/PrometheusScraper.java +++ b/features/prometheus-collector/src/main/java/org/opennms/netmgt/collectd/prometheus/PrometheusScraper.java @@ -27,25 +27,37 @@ import java.net.URI; import java.util.Map; +import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpHeaders; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.hawkular.agent.prometheus.PrometheusMetricsProcessor; +import org.hawkular.agent.prometheus.text.OpenMetricsProcessor; import org.hawkular.agent.prometheus.text.TextPrometheusMetricsProcessor; -import org.hawkular.agent.prometheus.walkers.MetricCollectingWalker; +import org.hawkular.agent.prometheus.walkers.PrometheusMetricsWalker; import org.opennms.core.utils.ParameterMap; import org.opennms.core.web.HttpClientWrapper; import org.opennms.netmgt.collection.api.ServiceParameters; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.net.MediaType; public class PrometheusScraper { private static final int DEFAULT_RETRY_COUNT = 2; private static final int DEFAULT_SO_TIMEOUT_MS = 10000; + + private static final Logger LOGGER = LoggerFactory.getLogger(PrometheusScraper.class); private static final String HEADER_PREFIX_PARM_KEY = "header-"; - public static final String DEFAULT_ACCEPT_HEADER = "application/openmetrics-text; version=0.0.1,text/plain;version=0.0.4;q=0.5,*/*;q=0.1"; + public static final String DEFAULT_ACCEPT_HEADER = "application/openmetrics-text;version=1.0.0, text/plain;version=0.0.4;q=0.5, */*;q=0.1"; + + //Be lenient with the OpenMetrics Content-Type header value and don't include the version + static final MediaType OPENMETRICS_LENIENT = MediaType.parse("application/openmetrics-text"); + static final MediaType PROMETHEUS_V0_0_4 = MediaType.parse("text/plain;version=0.0.4"); - public static void scrape(URI uri, Map parameters, MetricCollectingWalker walker) throws IOException { + public static void scrape(URI uri, Map parameters, PrometheusMetricsWalker walker) throws IOException { try (HttpClientWrapper httpClientWrapper = createHttpClientFromParmMap(parameters)) { final HttpGet get = new HttpGet(uri); get.setHeader(HttpHeaders.ACCEPT, DEFAULT_ACCEPT_HEADER); @@ -61,13 +73,35 @@ public static void scrape(URI uri, Map parameters, MetricCollect final HttpEntity entity = response.getEntity(); if (entity == null) { throw new IOException("No HTTP response entity from URL " + uri); + } + MediaType mediaType = PROMETHEUS_V0_0_4; // default: header absent or unparseable + Header contentType = response.getFirstHeader(HttpHeaders.CONTENT_TYPE); + if (contentType != null) { + try { + mediaType = MediaType.parse(contentType.getValue()); + } + catch (IllegalArgumentException e) { + LOGGER.warn("Invalid Content-Type '{}' from {} - assuming Prometheus text format", + contentType.getValue(), uri); + } + } else { + LOGGER.warn("No Content-Type header from {} - assuming Prometheus text format", uri); + } + PrometheusMetricsProcessor processor; + if(mediaType.is(OPENMETRICS_LENIENT)) { + LOGGER.debug("Processing response as OpenMetrics v1 format"); + processor = new OpenMetricsProcessor(entity.getContent(), walker); + } + else { + LOGGER.debug("Processing response as Prometheus text format (v0.0.4)"); + processor = new TextPrometheusMetricsProcessor(entity.getContent(), walker); } - PrometheusMetricsProcessor processor = new TextPrometheusMetricsProcessor(entity.getContent(), walker); processor.walk(); } } } + public static HttpClientWrapper createHttpClientFromParmMap(Map parameters) { // Timeouts and retries HttpClientWrapper clientWrapper = HttpClientWrapper.create() diff --git a/features/prometheus-collector/src/main/java/org/opennms/netmgt/dao/prometheus/PrometheusDataCollectionConfigDaoJaxb.java b/features/prometheus-collector/src/main/java/org/opennms/netmgt/dao/prometheus/PrometheusDataCollectionConfigDaoJaxb.java index 52603f848135..59cf78b39201 100644 --- a/features/prometheus-collector/src/main/java/org/opennms/netmgt/dao/prometheus/PrometheusDataCollectionConfigDaoJaxb.java +++ b/features/prometheus-collector/src/main/java/org/opennms/netmgt/dao/prometheus/PrometheusDataCollectionConfigDaoJaxb.java @@ -33,7 +33,6 @@ import org.opennms.netmgt.config.prometheus.Collection; import org.opennms.netmgt.config.prometheus.Group; import org.opennms.netmgt.config.prometheus.PrometheusDatacollectionConfig; -import org.opennms.netmgt.dao.prometheus.PrometheusDataCollectionConfigDao; public class PrometheusDataCollectionConfigDaoJaxb extends AbstractMergingJaxbConfigDao implements PrometheusDataCollectionConfigDao { diff --git a/features/prometheus-collector/src/test/java/org/hawkular/agent/prometheus/text/AssertUtils.java b/features/prometheus-collector/src/test/java/org/hawkular/agent/prometheus/text/AssertUtils.java new file mode 100644 index 000000000000..2cd13ed8d345 --- /dev/null +++ b/features/prometheus-collector/src/test/java/org/hawkular/agent/prometheus/text/AssertUtils.java @@ -0,0 +1,106 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.hawkular.agent.prometheus.text; + +import static org.junit.Assert.assertEquals; + +import java.util.List; +import java.util.Map; + +import org.hawkular.agent.prometheus.types.Counter; +import org.hawkular.agent.prometheus.types.Gauge; +import org.hawkular.agent.prometheus.types.Metric; +import org.hawkular.agent.prometheus.types.MetricFamily; +import org.hawkular.agent.prometheus.types.MetricType; +import org.hawkular.agent.prometheus.types.Summary; + +public class AssertUtils { + + public static void assertMetricFamily(MetricFamily expectedFamily, MetricFamily metricFamily) { + assertEquals(expectedFamily.getName(), metricFamily.getName()); + assertEquals(expectedFamily.getHelp(), metricFamily.getHelp()); + assertEquals(expectedFamily.getType(), metricFamily.getType()); + List expectedMetrics = expectedFamily.getMetrics(); + List metrics = metricFamily.getMetrics(); + assertEquals(expectedMetrics.size(), metrics.size()); + + for (int i = 0; i < expectedMetrics.size(); i++) { + Metric expectedMetric = expectedMetrics.get(i); + Metric metric = metrics.get(i); + assertBaseMetric(expectedMetric, metric, expectedFamily.getName()); + MetricType type = expectedFamily.getType(); + switch (type) { + case COUNTER: + assertCounter((Counter) expectedMetric, metric); + break; + case GAUGE: + case UNKNOWN: + assertGauge((Gauge) expectedMetric, metric); + break; + case SUMMARY: + assertSummary((Summary) expectedMetric, metric); + break; + default: + throw new IllegalArgumentException("Unsupported metric type: " + type); + } + } + } + + public static void assertBaseMetric(Metric expectedMetric, Metric metric, String name) { + assertEquals(name, metric.getName()); + if(expectedMetric.getLabels() != null) { + assertEquals(expectedMetric.getLabels().size(), metric.getLabels().size()); + for (Map.Entry expectedLabel : expectedMetric.getLabels().entrySet()) { + assertEquals(expectedLabel.getValue(), metric.getLabels().get(expectedLabel.getKey())); + } + } + optionalAssertEquals(expectedMetric.getTimestamp(), metric.getTimestamp()); + optionalAssertEquals(expectedMetric.getCreated(), metric.getCreated()); + } + + public static void optionalAssertEquals(Object expected, Object actual) { + if (expected != null) { + assertEquals(expected, actual); + } + } + + public static void assertCounter(Counter expectedMetric, Metric metric) { + assertEquals(Counter.class, metric.getClass()); + Counter counter = (Counter) metric; + assertEquals(expectedMetric.getValue(), counter.getValue(), 0.001); + } + + public static void assertGauge(Gauge expectedMetric, Metric metric) { + assertEquals(Gauge.class, metric.getClass()); + Gauge gauge = (Gauge) metric; + assertEquals(expectedMetric.getValue(), gauge.getValue(), 0.001); + } + + public static void assertSummary(Summary expectedMetric, Metric metric) { + assertEquals(Summary.class, metric.getClass()); + Summary summary = (Summary) metric; + long sampleCount = summary.getSampleCount(); + double sampleSum = summary.getSampleSum(); + assertEquals(sampleCount, expectedMetric.getSampleCount()); + assertEquals(sampleSum, expectedMetric.getSampleSum(), 0.001); + } +} diff --git a/features/prometheus-collector/src/test/java/org/hawkular/agent/prometheus/text/OpenMetricsTextV1Test.java b/features/prometheus-collector/src/test/java/org/hawkular/agent/prometheus/text/OpenMetricsTextV1Test.java new file mode 100644 index 000000000000..86e1e2507cb4 --- /dev/null +++ b/features/prometheus-collector/src/test/java/org/hawkular/agent/prometheus/text/OpenMetricsTextV1Test.java @@ -0,0 +1,160 @@ + /* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.hawkular.agent.prometheus.text; + +import static org.hawkular.agent.prometheus.text.AssertUtils.assertMetricFamily; +import static org.junit.Assert.assertNull; + +import java.io.IOException; +import java.io.InputStream; +import java.time.Instant; +import java.util.List; +import java.util.Map; + +import org.hawkular.agent.prometheus.types.Counter; +import org.hawkular.agent.prometheus.types.Gauge; +import org.hawkular.agent.prometheus.types.MetricFamily; +import org.hawkular.agent.prometheus.types.MetricType; +import org.hawkular.agent.prometheus.types.Summary; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + * + */ +public class OpenMetricsTextV1Test { + + private static final Logger LOGGER = LoggerFactory.getLogger(OpenMetricsTextV1Test.class); + + @Test + public void testOpenMetrics() throws IOException { + + MetricFamily gauge = new MetricFamily.Builder().setName("go_goroutines") + .setType(MetricType.GAUGE) + .setHelp("Number of goroutines that currently exist.") + .addMetric(Gauge.builder().setName("go_goroutines").setValue(69).build()) + .build(); + + + MetricFamily counter1 = new MetricFamily.Builder().setName("process_cpu_seconds") + .setType(MetricType.COUNTER) + .setHelp("Total user and system CPU time spent in seconds.") + .addMetric(Counter.builder().setName("process_cpu_seconds").setValue(4.20072246e+06).build()) + .build(); + + + Instant sampleTs1 = Instant.ofEpochSecond(1488452300L); + + MetricFamily counter2 = new MetricFamily.Builder().setName("http_requests") + .setType(MetricType.COUNTER) + .setHelp("Total number of HTTP requests made.") + .addMetric(Counter.builder().setName("http_requests") + .addLabels(Map.of("method", "get", "code", "200")) + .setValue(1027) + .setTimestamp(sampleTs1).build()) + .addMetric(Counter.builder().setName("http_requests") + .addLabels(Map.of("method", "post", "code", "200")) + .setValue(32) + .setTimestamp(sampleTs1).build()) + .build(); + + + Instant sampleTs2 = Instant.ofEpochSecond(1520879607L); + Instant createdTs = Instant.ofEpochSecond(1605281325L); + + MetricFamily counter3 = new MetricFamily.Builder().setName("foo") + .setType(MetricType.COUNTER) + .setHelp("") + .addMetric(Counter.builder().setName("foo") + .addLabel("color", "red") + .setValue(17) + .setTimestamp(sampleTs2) + .setCreated(createdTs).build()) + .addMetric(Counter.builder().setName("foo") + .addLabel("color", "green") + .setValue(10) + .setTimestamp(sampleTs2) + .setCreated(createdTs).build()) + .build(); + + + MetricFamily summary = new MetricFamily.Builder().setName("acme_http_router_request_seconds") + .setType(MetricType.SUMMARY) + .setHelp("Latency though all of ACME's HTTP request router.") + .addMetric(Summary.builder() + .setName("acme_http_router_request_seconds") + .addLabels(Map.of("path", "/api/v1", "method", "GET")) + .setSampleSum(9036.32) + .setSampleCount(807283) + .setCreated(Instant.ofEpochSecond(1605281325L)) + .build()) + .addMetric(Summary.builder() + .setName("acme_http_router_request_seconds") + .addLabels(Map.of("path", "/api/v2", "method", "POST")) + .setSampleSum(479.3) + .setSampleCount(34) + .setCreated(Instant.ofEpochSecond(1605281325L)) + .build() + ) + .build(); + + MetricFamily unknownMetric = new MetricFamily.Builder().setName("unknown_metric_1") + .setType(MetricType.UNKNOWN) + .setHelp("A metric with an unknown type") + .addMetric(Gauge.builder().setName("unknown_metric_1").setValue(42).build()) + .build(); + + MetricFamily untyped1 = new MetricFamily.Builder().setName("unknown_metric_2") + .setType(MetricType.UNKNOWN) + .setHelp("Missing type, defaults to unknown") + .addMetric(Gauge.builder().setName("unknown_metric_2").setValue(55).build()) + .build(); + + MetricFamily untyped2 = new MetricFamily.Builder().setName("unknown_metric_3") + .setType(MetricType.UNKNOWN) + .setHelp("") + .addMetric(Gauge.builder().setName("unknown_metric_3").setValue(66).build()) + .build(); + + MetricFamily anotherGauge = new MetricFamily.Builder().setName("another_gauge") + .setType(MetricType.GAUGE) + .setHelp("Another gauge metric.") + .addMetric(Gauge.builder().setName("another_gauge").setValue(128).build()) + .build(); + + List expectedMetricFamilies = List.of(gauge, counter1, counter2, counter3, summary, unknownMetric, untyped1, untyped2, anotherGauge); + + try(InputStream input = PrometheusTextV0_0_4Test.class.getResourceAsStream("/openmetrics.txt")){ + TextPrometheusMetricDataParser parser = new TextPrometheusMetricDataParser(input, OpenMetricsFormatStrategy.INSTANCE); + for (MetricFamily expected : expectedMetricFamilies) { + LOGGER.info("Validating metric {}", expected.getName()); + MetricFamily metricFamily = parser.parse(); + assertMetricFamily(expected, metricFamily); + } + assertNull("Expected no more metric families", parser.parse()); + } + } + + +} diff --git a/features/prometheus-collector/src/test/java/org/hawkular/agent/prometheus/text/PrometheusTextV0_0_4Test.java b/features/prometheus-collector/src/test/java/org/hawkular/agent/prometheus/text/PrometheusTextV0_0_4Test.java new file mode 100644 index 000000000000..a149e183c85a --- /dev/null +++ b/features/prometheus-collector/src/test/java/org/hawkular/agent/prometheus/text/PrometheusTextV0_0_4Test.java @@ -0,0 +1,108 @@ + /* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.hawkular.agent.prometheus.text; + +import static org.hawkular.agent.prometheus.text.AssertUtils.assertMetricFamily; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; + +import org.hawkular.agent.prometheus.types.Counter; +import org.hawkular.agent.prometheus.types.MetricFamily; +import org.hawkular.agent.prometheus.types.MetricType; +import org.junit.Test; + +/** + * Validates PrometheusText0.0.4 format + * + */ +public class PrometheusTextV0_0_4Test { + + @Test + public void testSingleCounter() throws IOException { + MetricFamily counter1 = new MetricFamily.Builder().setName("http_requests_total") + .setType(MetricType.COUNTER) + .setHelp("Total number of HTTP requests made.") + .addMetric(Counter.builder().setName("http_requests_total") + .addLabels(Map.of("code", "200", "handler", "prometheus", "method", "get")) + .setValue(162030) + .build()) + .addMetric(Counter.builder().setName("http_requests_total") + .addLabels(Map.of("code", "200", "handler", "query", "method", "get")) + .setValue(40) + .build()) + .addMetric(Counter.builder().setName("http_requests_total") + .addLabels(Map.of("code", "200", "handler", "series", "method", "get")) + .setValue(Double.NaN) + .build()) + .addMetric(Counter.builder().setName("http_requests_total") + .addLabels(Map.of("code", "400", "handler", "query", "method", "get")) + .setValue(Double.POSITIVE_INFINITY) + .build()) + .addMetric(Counter.builder().setName("http_requests_total") + .addLabels(Map.of("code", "400", "handler", "series", "method", "get")) + .setValue(Double.NEGATIVE_INFINITY) + .build()) + .build(); + + try (InputStream input = PrometheusTextV0_0_4Test.class.getResourceAsStream("/prometheus-counter.txt")) { + TextPrometheusMetricDataParser parser = new TextPrometheusMetricDataParser(input, PrometheusTextFormatStrategy.INSTANCE); + MetricFamily metricFamily = parser.parse(); + assertMetricFamily(counter1, metricFamily); + } + } + + @Test + public void testThreeCounters() throws IOException { + MetricFamily counter1 = new MetricFamily.Builder().setName("one_counter_total") + .setType(MetricType.COUNTER) + .setHelp("This is the first") + .addMetric(Counter.builder().setName("one_counter_total").setValue(111).build()) + .build(); + + MetricFamily counter2 = new MetricFamily.Builder().setName("two_counter_total") + .setType(MetricType.COUNTER) + .setHelp("This is the second") + .addMetric(Counter.builder().setName("two_counter_total").setValue(222).build()) + .build(); + + MetricFamily counter3 = new MetricFamily.Builder().setName("three_counter_total") + .setType(MetricType.COUNTER) + .setHelp("This is the third with type specified first") + .addMetric(Counter.builder().setName("three_counter_total").setValue(333).build()) + .build(); + + try (InputStream input = PrometheusTextV0_0_4Test.class.getResourceAsStream("/prometheus-three-counters.txt")) { + TextPrometheusMetricDataParser parser = new TextPrometheusMetricDataParser(input, PrometheusTextFormatStrategy.INSTANCE); + + MetricFamily metricFamily1 = parser.parse(); + assertMetricFamily(counter1, metricFamily1); + + MetricFamily metricFamily2 = parser.parse(); + assertMetricFamily(counter2, metricFamily2); + + MetricFamily metricFamily3 = parser.parse(); + assertMetricFamily(counter3, metricFamily3); + } + } +} diff --git a/features/prometheus-collector/src/test/java/org/opennnms/netmgt/collectd/prometheus/PrometheusCollectorIT.java b/features/prometheus-collector/src/test/java/org/opennnms/netmgt/collectd/prometheus/PrometheusCollectorIT.java index 14a8fef83d5a..d67618b340d2 100644 --- a/features/prometheus-collector/src/test/java/org/opennnms/netmgt/collectd/prometheus/PrometheusCollectorIT.java +++ b/features/prometheus-collector/src/test/java/org/opennnms/netmgt/collectd/prometheus/PrometheusCollectorIT.java @@ -25,6 +25,7 @@ import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static java.util.Collections.singletonList; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; @@ -92,12 +93,20 @@ public class PrometheusCollectorIT { public void setUp() { stubFor(get(urlEqualTo("/metrics")) .willReturn(aResponse() - .withHeader("Content-Type", "Content-Type: text/plain; version=0.0.4") + .withHeader("Content-Type", "text/plain; version=0.0.4") .withBodyFile("linux.metrics"))); stubFor(get(urlEqualTo("/nephron_on_flink")) .willReturn(aResponse() - .withHeader("Content-Type", "Content-Type: text/plain; version=0.0.4") + .withHeader("Content-Type", "text/plain; version=0.0.4") .withBodyFile("flink.metrics"))); + stubFor(get(urlEqualTo("/actuator/prometheus")) + .willReturn(aResponse() + .withHeader("Content-Type", "application/openmetrics-text;version=1.0.0") + .withBodyFile("open.metrics"))); + stubFor(get(urlEqualTo("/open_noncompliant")) + .willReturn(aResponse() + .withHeader("Content-Type", "application/openmetrics-text") + .withBodyFile("open_noncompliant.metrics"))); } @Test @@ -419,6 +428,77 @@ public void canGatherCpuMetrics() { "0/nodeExporterCPU/cpu7/node-exporter-cpu/system[null,200.86]", "0/nodeExporterCPU/cpu7/node-exporter-cpu/user[null,1203.71]"), collectionSetKeys); } + + @Test + public void canGatherOpenMetricsGauges() { + Collection collection = new Collection(); + + Group executorPool = new Group(); + executorPool.setName("executor-pool"); + executorPool.setFilterExp("name matches 'executor_pool_size_threads'"); + executorPool.setResourceType("node"); + + NumericAttribute executorPoolSize = new NumericAttribute(); + executorPoolSize.setAliasExp("labels[name]"); + executorPool.getNumericAttribute().add(executorPoolSize); + + // Collect! + CollectionSet collectionSet = collectOpenMetrics(collection,executorPool); + + // Verify + List collectionSetKeys = CollectionSetUtils.flatten(collectionSet); + assertThat(collectionSetKeys, hasSize(1)); + assertThat(collectionSetKeys, hasItem("0/executor-pool/taskExecutor[null,0.0]")); + } + + @Test + public void canGatherOpenMetricsCounters() { + Collection collection = new Collection(); + + Group logbackEvents = new Group(); + logbackEvents.setName("logback-events"); + logbackEvents.setFilterExp("name matches 'logback_events'"); + logbackEvents.setGroupByExp("labels[level]"); + logbackEvents.setResourceType("node"); + + NumericAttribute logbackEventCount = new NumericAttribute(); + logbackEventCount.setAliasExp("labels[level]"); + logbackEvents.getNumericAttribute().add(logbackEventCount); + + // Collect! + CollectionSet collectionSet = collectOpenMetrics(collection,logbackEvents); + + // Verify + List collectionSetKeys = CollectionSetUtils.flatten(collectionSet); + assertThat(collectionSetKeys, hasSize(5)); + assertThat(collectionSetKeys, hasItem("0/logback-events/debug[null,2.0]")); + assertThat(collectionSetKeys, hasItem("0/logback-events/error[null,9170.0]")); + assertThat(collectionSetKeys, hasItem("0/logback-events/info[null,251.0]")); + assertThat(collectionSetKeys, hasItem("0/logback-events/trace[null,1.0]")); + assertThat(collectionSetKeys, hasItem("0/logback-events/warn[null,2876.0]")); + } + + @Test + public void canGatherOpenMetricsNonCompliant() { + Collection collection = new Collection(); + + Group executorPool = new Group(); + executorPool.setName("executor-pool"); + executorPool.setFilterExp("name matches 'executor_pool_size_threads'"); + executorPool.setResourceType("node"); + + NumericAttribute executorPoolSize = new NumericAttribute(); + executorPoolSize.setAliasExp("labels[name]"); + executorPool.getNumericAttribute().add(executorPoolSize); + + // Collect! + CollectionSet collectionSet = collect(collection,singletonList(executorPool), "open_noncompliant"); + + // Verify + List collectionSetKeys = CollectionSetUtils.flatten(collectionSet); + assertThat(collectionSetKeys, hasSize(1)); + assertThat(collectionSetKeys, hasItem("0/executor-pool/taskExecutor[null,0.0]")); + } private CollectionSet collect(Collection collection, List groups) { return collect(collection, groups, "metrics"); @@ -427,6 +507,10 @@ private CollectionSet collect(Collection collection, List groups) { private CollectionSet collectNephronMetrics(Collection collection, Group... groups) { return collect(collection, Arrays.asList(groups), "nephron_on_flink"); } + + private CollectionSet collectOpenMetrics(Collection collection, Group... groups) { + return collect(collection, Arrays.asList(groups), "actuator/prometheus"); + } private CollectionSet collect(Collection collection, List groups, String path) { // Create the agent diff --git a/features/prometheus-collector/src/test/resources/__files/open.metrics b/features/prometheus-collector/src/test/resources/__files/open.metrics new file mode 100644 index 000000000000..b10f1a3b1b22 --- /dev/null +++ b/features/prometheus-collector/src/test/resources/__files/open.metrics @@ -0,0 +1,14 @@ +# TYPE executor_pool_size_threads gauge +# HELP executor_pool_size_threads The current number of threads in the pool +executor_pool_size_threads{name="taskExecutor"} 0.0 +# TYPE jvm_classes_unloaded_classes counter +# HELP jvm_classes_unloaded_classes The total number of classes unloaded since the Java virtual machine has started execution +jvm_classes_unloaded_classes_total 1.0 +# TYPE logback_events counter +# HELP logback_events Number of log events that were enabled by the effective log level +logback_events_total{level="warn"} 2876.0 +logback_events_total{level="debug"} 2.0 +logback_events_total{level="error"} 9170.0 +logback_events_total{level="trace"} 1.0 +logback_events_total{level="info"} 251.0 +# EOF \ No newline at end of file diff --git a/features/prometheus-collector/src/test/resources/__files/open_noncompliant.metrics b/features/prometheus-collector/src/test/resources/__files/open_noncompliant.metrics new file mode 100644 index 000000000000..a3a3d94c0193 --- /dev/null +++ b/features/prometheus-collector/src/test/resources/__files/open_noncompliant.metrics @@ -0,0 +1,4 @@ +# TYPE executor_pool_size_threads +# HELP executor_pool_size_threads Missing the type, assumes unknown/gauge +executor_pool_size_threads{name="taskExecutor"} 0.0 +# EOF \ No newline at end of file diff --git a/features/prometheus-collector/src/test/resources/mocklogger.properties b/features/prometheus-collector/src/test/resources/mocklogger.properties new file mode 100644 index 000000000000..3e6bfd71f35f --- /dev/null +++ b/features/prometheus-collector/src/test/resources/mocklogger.properties @@ -0,0 +1,2 @@ +org.opennms.core.test.mockLogger.defaultLogLevel=debug +org.opennms.core.test.mockLogger.log.org=debug \ No newline at end of file diff --git a/features/prometheus-collector/src/test/resources/openmetrics.txt b/features/prometheus-collector/src/test/resources/openmetrics.txt new file mode 100644 index 000000000000..0a93ccf5f58d --- /dev/null +++ b/features/prometheus-collector/src/test/resources/openmetrics.txt @@ -0,0 +1,43 @@ +# TYPE go_goroutines gauge +# HELP go_goroutines Number of goroutines that currently exist. +go_goroutines 69 +# TYPE process_cpu_seconds counter +# UNIT process_cpu_seconds seconds +# HELP process_cpu_seconds Total user and system CPU time spent in seconds. +process_cpu_seconds_total 4.20072246e+06 +# TYPE http_requests counter +# UNIT http_requests requests +# HELP http_requests Total number of HTTP requests made. +http_requests_total{method="get",code="200"} 1027 1.488452300E9 +http_requests_total{method="post",code="200"} 32 1.488452300E9 +# TYPE foo counter +foo_total{color="red"} 17.0 1520879607.789 +foo_created{color="red"} 1605281325.0 +foo_total{color="green"} 10 1520879607.789 +foo_created{color="green"} 1605281325.0 +# TYPE acme_http_router_request_seconds summary +# UNIT acme_http_router_request_seconds seconds +# HELP acme_http_router_request_seconds Latency though all of ACME's HTTP request router. +acme_http_router_request_seconds_sum{path="/api/v1",method="GET"} 9036.32 +acme_http_router_request_seconds_count{path="/api/v1",method="GET"} 807283.0 +acme_http_router_request_seconds_created{path="/api/v1",method="GET"} 1605281325.0 +acme_http_router_request_seconds_sum{path="/api/v2",method="POST"} 479.3 +acme_http_router_request_seconds_count{path="/api/v2",method="POST"} 34.0 +acme_http_router_request_seconds_created{path="/api/v2",method="POST"} 1605281325.0 +# TYPE unknown_metric_1 unknown +# HELP unknown_metric_1 A metric with an unknown type +unknown_metric_1 42 +unknown_metric_1_garbage 99 +garbage 12 +more_garbage 3 +# HELP unknown_metric_2 Missing type, defaults to unknown +unknown_metric_2 55 +# TYPE unknown_metric_3 +unknown_metric_3 66 +# HELP acme_count A metric with an invalid type. Should be ignored +# TYPE acme_count invalid_type +acme_count 10 +# TYPE another_gauge gauge +# HELP another_gauge Another gauge metric. +another_gauge 128 +# EOF \ No newline at end of file From c8cd3b57e841799f9846d561d6e456c82fc63dcf Mon Sep 17 00:00:00 2001 From: jassuncao Date: Fri, 10 Jul 2026 09:34:33 +0100 Subject: [PATCH 2/2] NMS-19910: Add support for OpenMetrics v1 text format scraping Minor corrections . Updates the documenation for the Prometheus collector --- .../collectors/prometheus.adoc | 9 +++++++-- .../org/hawkular/agent/prometheus/text/LineParser.java | 2 +- .../prometheus/text/TextPrometheusMetricDataParser.java | 2 +- .../org/hawkular/agent/prometheus/types/MetricType.java | 4 ++++ 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/modules/reference/pages/performance-data-collection/collectors/prometheus.adoc b/docs/modules/reference/pages/performance-data-collection/collectors/prometheus.adoc index 9ace55555461..f59241e1e6bc 100644 --- a/docs/modules/reference/pages/performance-data-collection/collectors/prometheus.adoc +++ b/docs/modules/reference/pages/performance-data-collection/collectors/prometheus.adoc @@ -3,11 +3,16 @@ = PrometheusCollector :description: Learn how to configure the PrometheusCollector in OpenNMS {page-component-title} to collect performance data via HTTP(S) using the Prometheus Exposition format. -The PrometheusCollector collects performance metrics via HTTP(S) using the text-based https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-based-format[Prometheus Exposition format]. -Many applications have adopted it and it is in the process of being standardized in the https://openmetrics.io/[OpenMetrics] project. +The PrometheusCollector collects performance metrics via HTTP(S), supporting both the classic https://github.com/prometheus/docs/blob/main/docs/instrumenting/exposition_formats.md#prometheus-text-format[Prometheus Text Format] and the newer https://github.com/prometheus/docs/blob/main/docs/specs/om/open_metrics_spec.md[OpenMetrics 1.0 Text Format]. + +It can scrape endpoints that provide either format or both. When both are available, it automatically prioritizes OpenMetrics. This protocol negotiation is performed via quality/priority values in the HTTP header `Accept`. This collector provides tools for parsing and mapping the metrics to the collection model used by {page-component-title}. +=== Limitations + +The PrometheusCollector only processes a subset of the metric types defined in both formats, namely counters and gauges. Histograms and summaries are *not* supported and will be ignored. Metric timestamps are also ignored. + == Collector facts [options="autowidth"] diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/LineParser.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/LineParser.java index 263ec20f7745..1e5bc3976750 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/LineParser.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/LineParser.java @@ -26,7 +26,7 @@ import org.hawkular.agent.prometheus.Util; -class SampleLineParser { +class LineParser { private enum ParseState { NAME, diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextPrometheusMetricDataParser.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextPrometheusMetricDataParser.java index 3e53eaac8a21..4caba4984e37 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextPrometheusMetricDataParser.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextPrometheusMetricDataParser.java @@ -40,7 +40,7 @@ public class TextPrometheusMetricDataParser extends PrometheusMetricDataParser