diff --git a/docs/modules/reference/pages/performance-data-collection/collectors/prometheus.adoc b/docs/modules/reference/pages/performance-data-collection/collectors/prometheus.adoc index 1b819e443a4c..022af523c9b9 100644 --- a/docs/modules/reference/pages/performance-data-collection/collectors/prometheus.adoc +++ b/docs/modules/reference/pages/performance-data-collection/collectors/prometheus.adoc @@ -3,11 +3,16 @@ = PrometheusCollector :description: Learn how to configure the PrometheusCollector in OpenNMS {page-component-title} to collect performance data via HTTP(S) using the Prometheus Exposition format. -The PrometheusCollector collects performance metrics via HTTP(S) using the text-based https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-based-format[Prometheus Exposition format]. -Many applications have adopted it and it is in the process of being standardized in the https://openmetrics.io/[OpenMetrics] project. +The PrometheusCollector collects performance metrics via HTTP(S), supporting both the classic https://github.com/prometheus/docs/blob/main/docs/instrumenting/exposition_formats.md#prometheus-text-format[Prometheus Text Format] and the newer https://github.com/prometheus/docs/blob/main/docs/specs/om/open_metrics_spec.md[OpenMetrics 1.0 Text Format]. + +It can scrape endpoints that provide either format or both. When both are available, it automatically prioritizes OpenMetrics. This protocol negotiation is performed via quality/priority values in the HTTP header `Accept`. This collector provides tools for parsing and mapping the metrics to the collection model used by {page-component-title}. +=== Limitations + +The PrometheusCollector only processes a subset of the metric types defined in both formats, namely counters and gauges. Histograms and summaries are *not* supported and will be ignored. Metric timestamps are also ignored. + == Collector facts [options="autowidth"] diff --git a/features/prometheus-collector/pom.xml b/features/prometheus-collector/pom.xml index 50dfd9f04616..ad53ecd5deb0 100644 --- a/features/prometheus-collector/pom.xml +++ b/features/prometheus-collector/pom.xml @@ -142,5 +142,11 @@ 1.57 test + + javax.persistence + javax.persistence-api + 2.2 + test + diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/PrometheusMetricsProcessor.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/PrometheusMetricsProcessor.java index ea2ca4b1ae93..20ab67b33076 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/PrometheusMetricsProcessor.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/PrometheusMetricsProcessor.java @@ -96,7 +96,7 @@ public void walk() { ((org.hawkular.agent.prometheus.types.Histogram) metric), metricIndex); break; - case UNTYPED: + case UNKNOWN: // if we can't tell what it is, it's a gauge. log.debug("UNtyped metric: '{}'",metric); walker.walkGaugeMetric(convertedMetricFamily, (Gauge) metric, metricIndex); diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/Util.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/Util.java index 8603c9b9b56f..91eb74db5f23 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/Util.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/Util.java @@ -16,7 +16,18 @@ */ package org.hawkular.agent.prometheus; +import java.time.Instant; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class Util { + + private static final Logger LOGGER = LoggerFactory.getLogger(Util.class); + + private Util() { + // prevent instantiation + } public static double convertStringToDouble(String valueString) { double doubleValue; @@ -39,4 +50,19 @@ public static String convertDoubleToString(double value) { } return String.format("%f", value); } + + public static Instant tryConvertTimestamp(CharSequence timestamp) { + if (timestamp == null || timestamp.isEmpty()) { + return null; + } + try { + double timestampDouble = Double.parseDouble(timestamp.toString()); + long epochSecond = (long) timestampDouble; + return Instant.ofEpochSecond(epochSecond); + } + catch (NumberFormatException e) { + LOGGER.warn("Invalid timestamp: " + timestamp); + return null; + } + } } diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/LineParser.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/LineParser.java new file mode 100644 index 000000000000..1e5bc3976750 --- /dev/null +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/LineParser.java @@ -0,0 +1,203 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.hawkular.agent.prometheus.text; + +import java.util.LinkedHashMap; +import java.util.Map; + +import org.hawkular.agent.prometheus.Util; + +class LineParser { + + private enum ParseState { + NAME, + ENDOFNAME, + STARTOFLABELNAME, + LABELNAME, + LABELVALUEEQUALS, + LABELVALUEQUOTE, + LABELVALUE, + LABELVALUESLASH, + NEXTLABEL, + ENDOFLABELS, + VALUE, + TIMESTAMP + } + + private final StringBuilder name = new StringBuilder(); + private final StringBuilder labelname = new StringBuilder(); + private final StringBuilder labelvalue = new StringBuilder(); + private final StringBuilder value = new StringBuilder(); + private final StringBuilder timestamp = new StringBuilder(); + private final Map labels = new LinkedHashMap<>(); + private ParseState state; + + public void reset() { + name.setLength(0); + labelname.setLength(0); + labelvalue.setLength(0); + value.setLength(0); + timestamp.setLength(0); + labels.clear(); + state = ParseState.NAME; + } + + public TextSample parse(String line) { + reset(); + + for (int c = 0; c < line.length(); c++) { + char charAt = line.charAt(c); + + switch (state) { + case NAME: + if (charAt == '{') { + state = ParseState.STARTOFLABELNAME; + } else if (charAt == ' ' || charAt == '\t') { + state = ParseState.ENDOFNAME; + } else { + name.append(charAt); + } + break; + case ENDOFNAME: + if (charAt == ' ' || charAt == '\t') { + // do nothing + } else if (charAt == '{') { + state = ParseState.STARTOFLABELNAME; + } else { + value.append(charAt); + state = ParseState.VALUE; + } + break; + case STARTOFLABELNAME: + if (charAt == ' ' || charAt == '\t') { + // do nothing + } else if (charAt == '}') { + state = ParseState.ENDOFLABELS; + } else { + labelname.append(charAt); + state = ParseState.LABELNAME; + } + break; + case LABELNAME: + if (charAt == '=') { + state = ParseState.LABELVALUEQUOTE; + } else if (charAt == '}') { + state = ParseState.ENDOFLABELS; + } else if (charAt == ' ' || charAt == '\t') { + state = ParseState.LABELVALUEEQUALS; + } else { + labelname.append(charAt); + } + break; + case LABELVALUEEQUALS: + if (charAt == '=') { + state = ParseState.LABELVALUEQUOTE; + } else if (charAt == ' ' || charAt == '\t') { + // do nothing + } else { + throw new IllegalStateException("Invalid line: " + line); + } + break; + case LABELVALUEQUOTE: + if (charAt == '"') { + state = ParseState.LABELVALUE; + } else if (charAt == ' ' || charAt == '\t') { + // do nothing + } else { + throw new IllegalStateException("Invalid line: " + line); + } + break; + case LABELVALUE: + if (charAt == '\\') { + state = ParseState.LABELVALUESLASH; + } else if (charAt == '"') { + labels.put(labelname.toString(), labelvalue.toString()); + labelname.setLength(0); + labelvalue.setLength(0); + state = ParseState.NEXTLABEL; + } else { + labelvalue.append(charAt); + } + break; + case LABELVALUESLASH: + state = ParseState.LABELVALUE; + if (charAt == '\\') { + labelvalue.append('\\'); + } else if (charAt == 'n') { + labelvalue.append('\n'); + } else if (charAt == '"') { + labelvalue.append('"'); + } else { + labelvalue.append('\\').append(charAt); + } + break; + case NEXTLABEL: + if (charAt == ',') { + state = ParseState.LABELNAME; + } else if (charAt == '}') { + state = ParseState.ENDOFLABELS; + } else if (charAt == ' ' || charAt == '\t') { + // do nothing + } else { + throw new IllegalStateException("Invalid line: " + line); + } + break; + case ENDOFLABELS: + if (charAt == ' ' || charAt == '\t') { + // do nothing + } else { + value.append(charAt); + state = ParseState.VALUE; + } + break; + case VALUE: + if (charAt == ' ' || charAt == '\t') { + state = ParseState.TIMESTAMP; + } else { + value.append(charAt); + } + break; + case TIMESTAMP: + if (charAt == ' ' || charAt == '\t') { + return buildSample(line); + } + else { + timestamp.append(charAt); + } + break; + } + } + + return buildSample(line); + } + + private TextSample buildSample(String line) { + return new TextSample.Builder() + .setLine(line) + .setName(name.toString()) + .setValue(value.toString()) + .setTimestamp(Util.tryConvertTimestamp(timestamp)) + .addLabels(new LinkedHashMap<>(labels)) + .build(); + } + +} \ No newline at end of file diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/OpenMetricsFormatStrategy.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/OpenMetricsFormatStrategy.java new file mode 100644 index 000000000000..715bcbc9d491 --- /dev/null +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/OpenMetricsFormatStrategy.java @@ -0,0 +1,186 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.hawkular.agent.prometheus.text; + +import java.util.Map; + +import org.hawkular.agent.prometheus.Util; +import org.hawkular.agent.prometheus.types.Counter; +import org.hawkular.agent.prometheus.types.Gauge; +import org.hawkular.agent.prometheus.types.Histogram; +import org.hawkular.agent.prometheus.types.Metric; +import org.hawkular.agent.prometheus.types.MetricType; +import org.hawkular.agent.prometheus.types.Summary; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + * Strategy for processing OpenMetrics v1 text data. + *

+ * The strategy is very similar to the Prometheus one but counters have a mandatory _total suffix and there is supported for the "created" data sample. + *

+ * + */ +enum OpenMetricsFormatStrategy implements TextFormatStrategy { + INSTANCE; + + private static final Logger LOGGER = LoggerFactory.getLogger(OpenMetricsFormatStrategy.class); + + @Override + public TextSampleProcessor getSampleProcessor(MetricType type) { + + switch (type) { + case COUNTER: + return OpenMetricsFormatStrategy::handleCounterSample; + case GAUGE: + //treat UNKNOWN as gauge + case UNKNOWN: + return OpenMetricsFormatStrategy::handleGaugeSample; + case SUMMARY: + return OpenMetricsFormatStrategy::handleSummarySample; + case HISTOGRAM: + return OpenMetricsFormatStrategy::handleHistogramSample; + default: + throw new IllegalArgumentException("Unknown metric type: " + type); + } + } + + @Override + public SampleNameValidator getNameValidator(MetricType type, String familyName) { + + switch (type) { + case COUNTER: + return SampleNameValidator.matchingNameWithSuffixes(familyName, "_total", "_created"); + case GAUGE: + case UNKNOWN: + return SampleNameValidator.matchingNameWithSuffixes(familyName, "", "_created"); + case SUMMARY: + return SampleNameValidator.matchingNameWithSuffixes(familyName, "", "_count", "_sum", "_created"); + case HISTOGRAM: + return SampleNameValidator.matchingNameWithSuffixes(familyName, "_count", "_sum", "_bucket", "_created"); + default: + throw new IllegalArgumentException("Unknown metric type: " + type); + } + } + + + private static void handleCounterSample(Map, Metric.Builder> builders, String familyName, TextSample textSample) { + Counter.Builder cBuilder = (Counter.Builder) builders.get(textSample.getLabels()); + if (cBuilder == null) { + cBuilder = Counter.builder() + .setName(familyName) + .addLabels(textSample.getLabels()); + builders.put(textSample.getLabels(), cBuilder); + } + if (textSample.getName().endsWith("_created")) { + cBuilder.setCreated(Util.tryConvertTimestamp(textSample.getValue())); + } + else { + cBuilder + .setValue(Util.convertStringToDouble(textSample.getValue())) + .setTimestamp(textSample.getTimestamp()); + } + } + + private static void handleGaugeSample(Map, Metric.Builder> builders, String familyName, TextSample textSample) { + Gauge.Builder gBuilder = (Gauge.Builder) builders.get(textSample.getLabels()); + if (gBuilder == null) { + gBuilder = Gauge.builder() + .setName(familyName) + .addLabels(textSample.getLabels()); + builders.put(textSample.getLabels(), gBuilder); + } + if (textSample.getName().endsWith("_created")) { + gBuilder.setCreated(Util.tryConvertTimestamp(textSample.getValue())); + } + else { + gBuilder.setValue(Util.convertStringToDouble(textSample.getValue())) + .setTimestamp(textSample.getTimestamp()); + } + } + + private static void handleSummarySample(Map, Metric.Builder> builders, String familyName, TextSample textSample) { + // First we need to remove any existing quantile label since it isn't a "real" label. + // This is to ensure our lookup uses all but only "real" labels. + String quantileValue = textSample.getLabels().remove("quantile"); // may be null + + Summary.Builder sBuilder = (Summary.Builder) builders.get(textSample.getLabels()); + if (sBuilder == null) { + sBuilder = new Summary.Builder(); + sBuilder.setName(familyName); + sBuilder.addLabels(textSample.getLabels()); + builders.put(textSample.getLabels(), sBuilder); + } + if (textSample.getName().endsWith("_count")) { + sBuilder.setSampleCount((long)Util.convertStringToDouble(textSample.getValue())); + } else if (textSample.getName().endsWith("_sum")) { + sBuilder.setSampleSum(Util.convertStringToDouble(textSample.getValue())); + } + else if (textSample.getName().endsWith("_created")) { + sBuilder.setCreated(Util.tryConvertTimestamp(textSample.getValue())); + } + else { + // This must be a quantile sample + if (quantileValue == null) { + LOGGER.debug("Summary quantile sample is missing the 'quantile' label: {}", + textSample.getLine()); + } + sBuilder.addQuantile(Util.convertStringToDouble(quantileValue), + Util.convertStringToDouble(textSample.getValue())); + } + } + + private static void handleHistogramSample(Map, Metric.Builder> builders, String familyName, TextSample textSample) { + // Get the builder that we are using to build up the current metric. Remember we need to + // get the builder for this specific metric identified with a unique set of labels. + + // First we need to remove any existing le label since it isn't a "real" label. + // This is to ensure our lookup uses all but only "real" labels. + String bucket = textSample.getLabels().remove("le"); // may be null + + Histogram.Builder hBuilder = (Histogram.Builder) builders.get(textSample.getLabels()); + if (hBuilder == null) { + hBuilder = new Histogram.Builder(); + hBuilder.setName(familyName); + hBuilder.addLabels(textSample.getLabels()); + builders.put(textSample.getLabels(), hBuilder); + } + if (textSample.getName().endsWith("_count")) { + hBuilder.setSampleCount((long)Util.convertStringToDouble(textSample.getValue())); + } else if (textSample.getName().endsWith("_sum")) { + hBuilder.setSampleSum(Util.convertStringToDouble(textSample.getValue())); + } + else if (textSample.getName().endsWith("_created")) { + hBuilder.setCreated(Util.tryConvertTimestamp(textSample.getValue())); + } + else { + // This must be a bucket sample + if (bucket == null) { + throw new IllegalArgumentException("Histogram bucket sample is missing the 'le' label"); + } + hBuilder.addBucket(Util.convertStringToDouble(bucket), + (long)Util.convertStringToDouble(textSample.getValue())); + } + } + +} diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/OpenMetricsProcessor.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/OpenMetricsProcessor.java new file mode 100644 index 000000000000..00ede0ba434c --- /dev/null +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/OpenMetricsProcessor.java @@ -0,0 +1,48 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.hawkular.agent.prometheus.text; + +import java.io.InputStream; + +import org.hawkular.agent.prometheus.PrometheusMetricsProcessor; +import org.hawkular.agent.prometheus.types.MetricFamily; +import org.hawkular.agent.prometheus.walkers.PrometheusMetricsWalker; + +/** + * This will iterate over a list of Prometheus metrics that are given in OpenMetrics text data. + */ +public class OpenMetricsProcessor extends PrometheusMetricsProcessor { + public OpenMetricsProcessor(InputStream inputStream, PrometheusMetricsWalker theWalker) { + super(inputStream, theWalker); + } + + @Override + public TextPrometheusMetricDataParser createPrometheusMetricDataParser() { + return new TextPrometheusMetricDataParser(getInputStream(), OpenMetricsFormatStrategy.INSTANCE); + } + + @Override + protected MetricFamily convert(MetricFamily metricFamily) { + return metricFamily; // no conversion necessary - our text parser already uses the common api + } + +} diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/PrometheusTextFormatStrategy.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/PrometheusTextFormatStrategy.java new file mode 100644 index 000000000000..0555ab5d9bec --- /dev/null +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/PrometheusTextFormatStrategy.java @@ -0,0 +1,151 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.hawkular.agent.prometheus.text; + +import java.util.Map; + +import org.hawkular.agent.prometheus.Util; +import org.hawkular.agent.prometheus.types.Counter; +import org.hawkular.agent.prometheus.types.Gauge; +import org.hawkular.agent.prometheus.types.Histogram; +import org.hawkular.agent.prometheus.types.Metric; +import org.hawkular.agent.prometheus.types.MetricType; +import org.hawkular.agent.prometheus.types.Summary; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +enum PrometheusTextFormatStrategy implements TextFormatStrategy { + + INSTANCE; + + private static final Logger LOGGER = LoggerFactory.getLogger(PrometheusTextFormatStrategy.class); + + @Override + public TextSampleProcessor getSampleProcessor(MetricType type) { + + switch (type) { + case COUNTER: + return PrometheusTextFormatStrategy::handleCounterSample; + case GAUGE: + // treat UNKNOWN as gauge + case UNKNOWN: + return PrometheusTextFormatStrategy::handleGaugeSample; + case HISTOGRAM: + return PrometheusTextFormatStrategy::handleHistogramSample; + case SUMMARY: + return PrometheusTextFormatStrategy::handleSummarySample; + default: + throw new IllegalArgumentException("Unknown metric type: " + type); + } + } + + @Override + public SampleNameValidator getNameValidator(MetricType type, String familyName) { + switch (type) { + case COUNTER: + case GAUGE: + case UNKNOWN: + return SampleNameValidator.exactName(familyName); + case HISTOGRAM: + return SampleNameValidator.matchingNameWithSuffixes(familyName, "", "_count", "_sum", "_bucket"); + case SUMMARY: + return SampleNameValidator.matchingNameWithSuffixes(familyName, "", "_count", "_sum"); + default: + throw new IllegalArgumentException("Unknown metric type: " + type); + } + } + + static void handleCounterSample(Map, Metric.Builder> builders, String familyName, TextSample textSample) { + builders.put(textSample.getLabels(), + new Counter.Builder().setName(familyName) + .setValue(Util.convertStringToDouble(textSample.getValue())) + .addLabels(textSample.getLabels())); + } + + static void handleGaugeSample(Map, Metric.Builder> builders, String familyName, TextSample textSample) { + builders.put(textSample.getLabels(), + new Gauge.Builder().setName(familyName) + .setValue(Util.convertStringToDouble(textSample.getValue())) + .addLabels(textSample.getLabels())); + } + + static void handleSummarySample(Map, Metric.Builder> builders, String familyName, TextSample textSample) { + // Get the builder that we are using to build up the current metric. Remember we need to + // get the builder for this specific metric identified with a unique set of labels. + + // First we need to remove any existing quantile label since it isn't a "real" label. + // This is to ensure our lookup uses all but only "real" labels. + String quantileValue = textSample.getLabels().remove("quantile"); // may be null + + Summary.Builder sBuilder = (Summary.Builder) builders.get(textSample.getLabels()); + if (sBuilder == null) { + sBuilder = new Summary.Builder(); + sBuilder.setName(familyName); + sBuilder.addLabels(textSample.getLabels()); + builders.put(textSample.getLabels(), sBuilder); + } + if (textSample.getName().endsWith("_count")) { + sBuilder.setSampleCount((long)Util.convertStringToDouble(textSample.getValue())); + } else if (textSample.getName().endsWith("_sum")) { + sBuilder.setSampleSum(Util.convertStringToDouble(textSample.getValue())); + } else { + // This must be a quantile sample + if (quantileValue == null) { + LOGGER.debug("Summary quantile sample is missing the 'quantile' label: {}", + textSample.getLine()); + } + sBuilder.addQuantile(Util.convertStringToDouble(quantileValue), + Util.convertStringToDouble(textSample.getValue())); + } + } + + static void handleHistogramSample(Map, Metric.Builder> builders, String familyName, TextSample textSample) { + // Get the builder that we are using to build up the current metric. Remember we need to + // get the builder for this specific metric identified with a unique set of labels. + + // First we need to remove any existing le label since it isn't a "real" label. + // This is to ensure our lookup uses all but only "real" labels. + String bucket = textSample.getLabels().remove("le"); // may be null + + Histogram.Builder hBuilder = (Histogram.Builder) builders.get(textSample.getLabels()); + if (hBuilder == null) { + hBuilder = new Histogram.Builder(); + hBuilder.setName(familyName); + hBuilder.addLabels(textSample.getLabels()); + builders.put(textSample.getLabels(), hBuilder); + } + if (textSample.getName().endsWith("_count")) { + hBuilder.setSampleCount((long)Util.convertStringToDouble(textSample.getValue())); + } else if (textSample.getName().endsWith("_sum")) { + hBuilder.setSampleSum(Util.convertStringToDouble(textSample.getValue())); + } else { + // This must be a bucket sample + if (bucket == null) { + throw new IllegalArgumentException("Histogram bucket sample is missing the 'le' label"); + } + hBuilder.addBucket(Util.convertStringToDouble(bucket), + (long)Util.convertStringToDouble(textSample.getValue())); + } + } + + +} diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/SampleNameValidator.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/SampleNameValidator.java new file mode 100644 index 000000000000..f893dbbf3764 --- /dev/null +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/SampleNameValidator.java @@ -0,0 +1,60 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.hawkular.agent.prometheus.text; + +interface SampleNameValidator { + boolean isValid(String sampleName); + + public static SampleNameValidator rejectAll() { + return sampleName -> false; + } + + /** + * Returns a validator that only accepts sample names that exactly match the given family name. + * @param familyName + * @return + */ + public static SampleNameValidator exactName(String familyName) { + return sampleName -> familyName.equals(sampleName); + } + + /** + * Returns a validator that only accepts sample names that start with the given family name and end with one of the given suffixes. + * @param familyName + * @param suffixes + * @return + */ + public static SampleNameValidator matchingNameWithSuffixes(String familyName, String... suffixes) { + return sampleName -> { + if(!sampleName.startsWith(familyName)) { + return false; + } + String suffix = sampleName.substring(familyName.length()); + for (String s : suffixes) { + if (s.equals(suffix)) { + return true; + } + } + return false; + }; + } +} diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextFormatStrategy.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextFormatStrategy.java new file mode 100644 index 000000000000..b02dd40f858d --- /dev/null +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextFormatStrategy.java @@ -0,0 +1,33 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.hawkular.agent.prometheus.text; + +import org.hawkular.agent.prometheus.types.MetricType; + +/** + * Strategy for processing text samples based on the metric type. + * + */ +interface TextFormatStrategy { + TextSampleProcessor getSampleProcessor(MetricType type); + SampleNameValidator getNameValidator(MetricType type, String familyName); +} diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextPrometheusMetricDataParser.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextPrometheusMetricDataParser.java index a03f5548b44a..4caba4984e37 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextPrometheusMetricDataParser.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextPrometheusMetricDataParser.java @@ -24,15 +24,11 @@ import java.util.List; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.hawkular.agent.prometheus.PrometheusMetricDataParser; -import org.hawkular.agent.prometheus.Util; -import org.hawkular.agent.prometheus.types.Counter; -import org.hawkular.agent.prometheus.types.Gauge; -import org.hawkular.agent.prometheus.types.Histogram; import org.hawkular.agent.prometheus.types.Metric; import org.hawkular.agent.prometheus.types.MetricFamily; import org.hawkular.agent.prometheus.types.MetricType; -import org.hawkular.agent.prometheus.types.Summary; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,6 +39,9 @@ public class TextPrometheusMetricDataParser extends PrometheusMetricDataParser allowedNames = new ArrayList<>(); public List textSamples = new ArrayList<>(); + public SampleNameValidator sampleNameValidator = SampleNameValidator.rejectAll(); // starts a fresh metric family public void clear() { name = ""; help = ""; type = null; - allowedNames.clear(); + sampleNameValidator = SampleNameValidator.rejectAll(); textSamples.clear(); } @@ -92,82 +92,12 @@ public void finishMetricFamily() { // For histogram metrics, we need to combine all bucket samples, sum, and count. Map, Metric.Builder> builders = new LinkedHashMap<>(); + + TextSampleProcessor textSampleProcessor = strategy.getSampleProcessor(type); for (TextSample textSample : textSamples) { try { - switch (type) { - case COUNTER: - builders.put(textSample.getLabels(), - new Counter.Builder().setName(name) - .setValue(Util.convertStringToDouble(textSample.getValue())) - .addLabels(textSample.getLabels())); - break; - case UNTYPED: - //treat UNTYPED as gauge - case GAUGE: - builders.put(textSample.getLabels(), - new Gauge.Builder().setName(name) - .setValue(Util.convertStringToDouble(textSample.getValue())) - .addLabels(textSample.getLabels())); - break; - case SUMMARY: - // Get the builder that we are using to build up the current metric. Remember we need to - // get the builder for this specific metric identified with a unique set of labels. - - // First we need to remove any existing quantile label since it isn't a "real" label. - // This is to ensure our lookup uses all but only "real" labels. - String quantileValue = textSample.getLabels().remove("quantile"); // may be null - - Summary.Builder sBuilder = (Summary.Builder) builders.get(textSample.getLabels()); - if (sBuilder == null) { - sBuilder = new Summary.Builder(); - builders.put(textSample.getLabels(), sBuilder); - } - sBuilder.setName(name); - sBuilder.addLabels(textSample.getLabels()); - if (textSample.getName().endsWith("_count")) { - sBuilder.setSampleCount((long)Util.convertStringToDouble(textSample.getValue())); - } else if (textSample.getName().endsWith("_sum")) { - sBuilder.setSampleSum(Util.convertStringToDouble(textSample.getValue())); - } else { - // This must be a quantile sample - if (quantileValue == null) { - log.debug("Summary quantile sample is missing the 'quantile' label: {}", - textSample.getLine()); - } - sBuilder.addQuantile(Util.convertStringToDouble(quantileValue), - Util.convertStringToDouble(textSample.getValue())); - } - break; - case HISTOGRAM: - // Get the builder that we are using to build up the current metric. Remember we need to - // get the builder for this specific metric identified with a unique set of labels. - - // First we need to remove any existing le label since it isn't a "real" label. - // This is to ensure our lookup uses all but only "real" labels. - String bucket = textSample.getLabels().remove("le"); // may be null - - Histogram.Builder hBuilder = (Histogram.Builder) builders.get(textSample.getLabels()); - if (hBuilder == null) { - hBuilder = new Histogram.Builder(); - builders.put(textSample.getLabels(), hBuilder); - } - hBuilder.setName(name); - hBuilder.addLabels(textSample.getLabels()); - if (textSample.getName().endsWith("_count")) { - hBuilder.setSampleCount((long)Util.convertStringToDouble(textSample.getValue())); - } else if (textSample.getName().endsWith("_sum")) { - hBuilder.setSampleSum(Util.convertStringToDouble(textSample.getValue())); - } else { - // This must be a bucket sample - if (bucket == null) { - throw new Exception("Histogram bucket sample is missing the 'le' label"); - } - hBuilder.addBucket(Util.convertStringToDouble(bucket), - (long)Util.convertStringToDouble(textSample.getValue())); - } - break; - } + textSampleProcessor.process(builders, name, textSample); } catch (Exception e) { log.debug("Error processing sample. This metric sample will be ignored: {}", textSample.getLine(), e); @@ -233,8 +163,8 @@ public MetricFamily parse() throws IOException { // start anew context.clear(); context.name = parts[2]; - context.type = MetricType.GAUGE; // default in case we don't get a TYPE - context.allowedNames.add(parts[2]); + context.type = MetricType.UNKNOWN; // default in case we don't get a TYPE + context.sampleNameValidator = strategy.getNameValidator(context.type, context.name); } if (parts.length == 4) { @@ -253,36 +183,22 @@ public MetricFamily parse() throws IOException { context.clear(); context.name = parts[2]; } - context.type = MetricType.valueOf(parts[3].toUpperCase()); - context.allowedNames.clear(); - switch (context.type) { - case COUNTER: - context.allowedNames.add(context.name); - break; - case GAUGE: - context.allowedNames.add(context.name); - break; - case UNTYPED: - context.allowedNames.add(context.name); - break; - case SUMMARY: - context.allowedNames.add(context.name + "_count"); - context.allowedNames.add(context.name + "_sum"); - context.allowedNames.add(context.name); - break; - case HISTOGRAM: - context.allowedNames.add(context.name + "_count"); - context.allowedNames.add(context.name + "_sum"); - context.allowedNames.add(context.name + "_bucket"); - break; + String type = parts.length < 4 ? "" : parts[3]; + context.type = StringUtils.isEmpty(type) ? MetricType.UNKNOWN : MetricType.tryParse(type); + if(context.type != null) { + context.sampleNameValidator = strategy.getNameValidator(context.type, context.name); } + else { + // we don't recognize this type, so we will ignore all samples for this family + context.clear(); + } } else { // ignore other tokens - probably a comment } } else { // parse the sample line that contains a single metric (or part of a metric as in summary/histo) - TextSample sample = parseSampleLine(line); - if (!context.allowedNames.contains(sample.getName())) { + TextSample sample = sampleLineParser.parse(line); + if (!context.sampleNameValidator.isValid(sample.getName())) { if (!context.name.isEmpty()) { // break and we'll finish the metric family we previously were building up this.lastLineReadFromStream = line; @@ -304,7 +220,7 @@ public MetricFamily parse() throws IOException { line = readLine(getInputStream()); } - if (!context.name.isEmpty()) { + if (!context.name.isEmpty() && context.type != null) { // finish the metric family we previously were building up context.finishMetricFamily(); } @@ -312,127 +228,6 @@ public MetricFamily parse() throws IOException { return context.finishedMetricFamily; } - private TextSample parseSampleLine(String line) { - // algorithm from parser.py - StringBuilder name = new StringBuilder(); - StringBuilder labelname = new StringBuilder(); - StringBuilder labelvalue = new StringBuilder(); - StringBuilder value = new StringBuilder(); - Map labels = new LinkedHashMap<>(); - - String state = "name"; - - for (int c = 0; c < line.length(); c++) { - char charAt = line.charAt(c); - if (state.equals("name")) { - if (charAt == '{') { - state = "startoflabelname"; - } else if (charAt == ' ' || charAt == '\t') { - state = "endofname"; - } else { - name.append(charAt); - } - } else if (state.equals("endofname")) { - if (charAt == ' ' || charAt == '\t') { - // do nothing - } else if (charAt == '{') { - state = "startoflabelname"; - } else { - value.append(charAt); - state = "value"; - } - } else if (state.equals("startoflabelname")) { - if (charAt == ' ' || charAt == '\t') { - // do nothing - } else if (charAt == '}') { - state = "endoflabels"; - } else { - labelname.append(charAt); - state = "labelname"; - } - } else if (state.equals("labelname")) { - if (charAt == '=') { - state = "labelvaluequote"; - } else if (charAt == '}') { - state = "endoflabels"; - } else if (charAt == ' ' || charAt == '\t') { - state = "labelvalueequals"; - } else { - labelname.append(charAt); - } - } else if (state.equals("labelvalueequals")) { - if (charAt == '=') { - state = "labelvaluequote"; - } else if (charAt == ' ' || charAt == '\t') { - // do nothing - } else { - throw new IllegalStateException("Invalid line: " + line); - } - } else if (state.equals("labelvaluequote")) { - if (charAt == '"') { - state = "labelvalue"; - } else if (charAt == ' ' || charAt == '\t') { - // do nothing - } else { - throw new IllegalStateException("Invalid line: " + line); - } - } else if (state.equals("labelvalue")) { - if (charAt == '\\') { - state = "labelvalueslash"; - } else if (charAt == '"') { - labels.put(labelname.toString(), labelvalue.toString()); - labelname.setLength(0); - labelvalue.setLength(0); - state = "nextlabel"; - } else { - labelvalue.append(charAt); - } - } else if (state.equals("labelvalueslash")) { - state = "labelvalue"; - if (charAt == '\\') { - labelvalue.append('\\'); - } else if (charAt == 'n') { - labelvalue.append('\n'); - } else if (charAt == '"') { - labelvalue.append('"'); - } else { - labelvalue.append('\\').append(charAt); - } - } else if (state.equals("nextlabel")) { - if (charAt == ',') { - state = "labelname"; - } else if (charAt == '}') { - state = "endoflabels"; - } else if (charAt == ' ' || charAt == '\t') { - // do nothing - } else { - throw new IllegalStateException("Invalid line: " + line); - } - } else if (state.equals("endoflabels")) { - if (charAt == ' ' || charAt == '\t') { - // do nothing - } else { - value.append(charAt); - state = "value"; - } - } else if (state.equals("value")) { - if (charAt == ' ' || charAt == '\t') { - break; // timestamps are NOT supported - ignoring - } else { - value.append(charAt); - } - } - } - - TextSample sample = new TextSample.Builder() - .setLine(line) - .setName(name.toString()) - .setValue(value.toString()) - .addLabels(labels).build(); - - return sample; - } - private String unescapeHelp(String text) { // algorithm from parser.py if (text == null || !text.contains("\\")) { diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextPrometheusMetricsProcessor.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextPrometheusMetricsProcessor.java index cb5777d21961..4063026b47a7 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextPrometheusMetricsProcessor.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextPrometheusMetricsProcessor.java @@ -32,7 +32,7 @@ public TextPrometheusMetricsProcessor(InputStream inputStream, PrometheusMetrics @Override public TextPrometheusMetricDataParser createPrometheusMetricDataParser() { - return new TextPrometheusMetricDataParser(getInputStream()); + return new TextPrometheusMetricDataParser(getInputStream(), PrometheusTextFormatStrategy.INSTANCE); } @Override diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextSampleProcessor.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextSampleProcessor.java new file mode 100644 index 000000000000..41d0edf99de1 --- /dev/null +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/text/TextSampleProcessor.java @@ -0,0 +1,37 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.hawkular.agent.prometheus.text; + +import java.util.Map; + +import org.hawkular.agent.prometheus.types.Metric; + +@FunctionalInterface +interface TextSampleProcessor { + /** + * Processes a Prometheus text sample and adds it to the given builders map. + * @param builders a map of builders for each metric family, keyed by a map of labels to the builder + * @param familyName the name of the metric family + * @param textSample the text sample to process + */ + void process(Map, Metric.Builder> builders, String familyName, TextSample textSample); +} diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Counter.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Counter.java index 732f911539f6..b84b9a557eb8 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Counter.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Counter.java @@ -42,9 +42,21 @@ public Counter(Builder builder) { public double getValue() { return value; } + + @Override + public String toString() { + + return "Counter [name=" + getName() + ", labels" + getLabels() + ", timestamp=" + getTimestamp() + ", created=" + getCreated() + + ", value=" + value + "]"; + } @Override public void visit(MetricVisitor visitor) { visitor.visitCounter(this); } + + public static Builder builder() { + + return new Builder(); + } } diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Gauge.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Gauge.java index b6fae94b35f6..e040000b7835 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Gauge.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Gauge.java @@ -43,8 +43,19 @@ public double getValue() { return value; } + @Override + public String toString() { + + return "Gauge [name=" + getName() + ", labels" + getLabels() + ", timestamp=" + getTimestamp() + ", created=" + getCreated() + + ", value=" + value + "]"; + } + @Override public void visit(MetricVisitor visitor) { visitor.visitGauge(this); } + + public static Builder builder() { + return new Builder(); + } } diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Histogram.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Histogram.java index 70eda6763628..cd1832a5c3e7 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Histogram.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Histogram.java @@ -110,6 +110,13 @@ public List getBuckets() { } return buckets; } + + @Override + public String toString() { + + return "Histogram [name=" + getName() + ", labels" + getLabels() + ", timestamp=" + getTimestamp() + ", created=" + getCreated() + + ", sampleCount=" + sampleCount + ", sampleSum=" + sampleSum + "]"; + } @Override public void visit(MetricVisitor visitor) { diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Metric.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Metric.java index 7f236313bdfc..d8fc5b1349d4 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Metric.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Metric.java @@ -16,6 +16,7 @@ */ package org.hawkular.agent.prometheus.types; +import java.time.Instant; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; @@ -28,6 +29,8 @@ public abstract class Metric { public abstract static class Builder> { private String name; private Map labels; + private Instant timestamp; + private Instant created; @SuppressWarnings("unchecked") public B setName(String name) { @@ -52,12 +55,26 @@ public B addLabels(Map map) { labels.putAll(map); return (B) this; } + + @SuppressWarnings("unchecked") + public B setTimestamp(Instant timestamp) { + this.timestamp = timestamp; + return (B) this; + } + + @SuppressWarnings("unchecked") + public B setCreated(Instant created) { + this.created = created; + return (B) this; + } public abstract T build(); } private final String name; private final Map labels; + private final Instant timestamp; + private final Instant created; protected Metric(Builder builder) { if (builder.name == null) { @@ -66,6 +83,8 @@ protected Metric(Builder builder) { this.name = builder.name; this.labels = builder.labels; + this.timestamp = builder.timestamp; + this.created = builder.created; } public String getName() { @@ -79,5 +98,20 @@ public Map getLabels() { return labels; } + public Instant getTimestamp() { + return timestamp; + } + + public Instant getCreated() { + + return created; + } + + @Override + public String toString() { + + return "Metric [name=" + name + ", labels=" + labels + ", timestamp=" + timestamp + ", created=" + created + "]"; + } + public abstract void visit(MetricVisitor visitor); } diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/MetricFamily.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/MetricFamily.java index a38b6e47e0a4..62147b37715a 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/MetricFamily.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/MetricFamily.java @@ -87,7 +87,7 @@ protected MetricFamily(Builder builder) { case HISTOGRAM: expectedMetricClassType = Histogram.class; break; - case UNTYPED: + case UNKNOWN: //treat untyped metrics as gauge expectedMetricClassType = Gauge.class; break; @@ -132,4 +132,9 @@ public List getMetrics() { } return metrics; } + + @Override + public String toString() { + return "MetricFamily [name=" + name + ", help=" + help + ", type=" + type + ", metrics=" + metrics + "]"; + } } diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/MetricType.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/MetricType.java index 72e267d915f3..1ca4ff2b3407 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/MetricType.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/MetricType.java @@ -18,5 +18,26 @@ package org.hawkular.agent.prometheus.types; public enum MetricType { - COUNTER, GAUGE, SUMMARY, HISTOGRAM, UNTYPED + COUNTER, GAUGE, SUMMARY, HISTOGRAM, UNKNOWN; + + /** + * Try to parse the given type string into a MetricType enum value. If the type is not recognized, it will return null. + * @param value the type string to parse. + * @return the MetricType enum value or null if not recognized + */ + public static MetricType tryParse(String value) { + if (value == null) { + return null; + } + try { + return MetricType.valueOf(value.toUpperCase()); + } + catch (IllegalArgumentException e) { + // legacy Prometheus v0.0.4 keyword, renamed to "unknown" in OpenMetrics + if ("untyped".equalsIgnoreCase(value)) { + return MetricType.UNKNOWN; + } + return null; + } + } } diff --git a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Summary.java b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Summary.java index 45b135823611..b902e41cf235 100644 --- a/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Summary.java +++ b/features/prometheus-collector/src/main/java/org/hawkular/agent/prometheus/types/Summary.java @@ -111,9 +111,20 @@ public List getQuantiles() { } return quantiles; } + + @Override + public String toString() { + + return "Summary [name=" + getName() + ", labels" + getLabels() + ", timestamp=" + getTimestamp() + ", created=" + getCreated() + + ", sampleCount=" + sampleCount + ", sampleSum=" + sampleSum + "]"; + } @Override public void visit(MetricVisitor visitor) { visitor.visitSummary(this); } + + public static Builder builder() { + return new Builder(); + } } diff --git a/features/prometheus-collector/src/main/java/org/opennms/netmgt/collectd/prometheus/PrometheusScraper.java b/features/prometheus-collector/src/main/java/org/opennms/netmgt/collectd/prometheus/PrometheusScraper.java index cb915ea90531..8bf5016b6919 100644 --- a/features/prometheus-collector/src/main/java/org/opennms/netmgt/collectd/prometheus/PrometheusScraper.java +++ b/features/prometheus-collector/src/main/java/org/opennms/netmgt/collectd/prometheus/PrometheusScraper.java @@ -29,25 +29,37 @@ import java.security.GeneralSecurityException; import java.util.Map; +import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpHeaders; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.hawkular.agent.prometheus.PrometheusMetricsProcessor; +import org.hawkular.agent.prometheus.text.OpenMetricsProcessor; import org.hawkular.agent.prometheus.text.TextPrometheusMetricsProcessor; -import org.hawkular.agent.prometheus.walkers.MetricCollectingWalker; +import org.hawkular.agent.prometheus.walkers.PrometheusMetricsWalker; import org.opennms.core.utils.ParameterMap; import org.opennms.core.web.HttpClientWrapper; import org.opennms.netmgt.collection.api.ServiceParameters; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.net.MediaType; public class PrometheusScraper { private static final int DEFAULT_RETRY_COUNT = 2; private static final int DEFAULT_SO_TIMEOUT_MS = 10000; + + private static final Logger LOGGER = LoggerFactory.getLogger(PrometheusScraper.class); private static final String HEADER_PREFIX_PARM_KEY = "header-"; - public static final String DEFAULT_ACCEPT_HEADER = "application/openmetrics-text; version=0.0.1,text/plain;version=0.0.4;q=0.5,*/*;q=0.1"; + public static final String DEFAULT_ACCEPT_HEADER = "application/openmetrics-text;version=1.0.0, text/plain;version=0.0.4;q=0.5, */*;q=0.1"; + + //Be lenient with the OpenMetrics Content-Type header value and don't include the version + static final MediaType OPENMETRICS_LENIENT = MediaType.parse("application/openmetrics-text"); + static final MediaType PROMETHEUS_V0_0_4 = MediaType.parse("text/plain;version=0.0.4"); - public static void scrape(URI uri, Map parameters, MetricCollectingWalker walker) throws IOException { + public static void scrape(URI uri, Map parameters, PrometheusMetricsWalker walker) throws IOException { try (HttpClientWrapper httpClientWrapper = createHttpClientFromParmMap(parameters)) { final HttpGet get = new HttpGet(uri); get.setHeader(HttpHeaders.ACCEPT, DEFAULT_ACCEPT_HEADER); @@ -63,13 +75,35 @@ public static void scrape(URI uri, Map parameters, MetricCollect final HttpEntity entity = response.getEntity(); if (entity == null) { throw new IOException("No HTTP response entity from URL " + uri); + } + MediaType mediaType = PROMETHEUS_V0_0_4; // default: header absent or unparseable + Header contentType = response.getFirstHeader(HttpHeaders.CONTENT_TYPE); + if (contentType != null) { + try { + mediaType = MediaType.parse(contentType.getValue()); + } + catch (IllegalArgumentException e) { + LOGGER.warn("Invalid Content-Type '{}' from {} - assuming Prometheus text format", + contentType.getValue(), uri); + } + } else { + LOGGER.warn("No Content-Type header from {} - assuming Prometheus text format", uri); + } + PrometheusMetricsProcessor processor; + if(mediaType.is(OPENMETRICS_LENIENT)) { + LOGGER.debug("Processing response as OpenMetrics v1 format"); + processor = new OpenMetricsProcessor(entity.getContent(), walker); + } + else { + LOGGER.debug("Processing response as Prometheus text format (v0.0.4)"); + processor = new TextPrometheusMetricsProcessor(entity.getContent(), walker); } - PrometheusMetricsProcessor processor = new TextPrometheusMetricsProcessor(entity.getContent(), walker); processor.walk(); } } } + public static HttpClientWrapper createHttpClientFromParmMap(Map parameters) throws IOException { // Timeouts and retries HttpClientWrapper clientWrapper = HttpClientWrapper.create() diff --git a/features/prometheus-collector/src/main/java/org/opennms/netmgt/dao/prometheus/PrometheusDataCollectionConfigDaoJaxb.java b/features/prometheus-collector/src/main/java/org/opennms/netmgt/dao/prometheus/PrometheusDataCollectionConfigDaoJaxb.java index 52603f848135..59cf78b39201 100644 --- a/features/prometheus-collector/src/main/java/org/opennms/netmgt/dao/prometheus/PrometheusDataCollectionConfigDaoJaxb.java +++ b/features/prometheus-collector/src/main/java/org/opennms/netmgt/dao/prometheus/PrometheusDataCollectionConfigDaoJaxb.java @@ -33,7 +33,6 @@ import org.opennms.netmgt.config.prometheus.Collection; import org.opennms.netmgt.config.prometheus.Group; import org.opennms.netmgt.config.prometheus.PrometheusDatacollectionConfig; -import org.opennms.netmgt.dao.prometheus.PrometheusDataCollectionConfigDao; public class PrometheusDataCollectionConfigDaoJaxb extends AbstractMergingJaxbConfigDao implements PrometheusDataCollectionConfigDao { diff --git a/features/prometheus-collector/src/test/java/org/hawkular/agent/prometheus/text/AssertUtils.java b/features/prometheus-collector/src/test/java/org/hawkular/agent/prometheus/text/AssertUtils.java new file mode 100644 index 000000000000..2cd13ed8d345 --- /dev/null +++ b/features/prometheus-collector/src/test/java/org/hawkular/agent/prometheus/text/AssertUtils.java @@ -0,0 +1,106 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.hawkular.agent.prometheus.text; + +import static org.junit.Assert.assertEquals; + +import java.util.List; +import java.util.Map; + +import org.hawkular.agent.prometheus.types.Counter; +import org.hawkular.agent.prometheus.types.Gauge; +import org.hawkular.agent.prometheus.types.Metric; +import org.hawkular.agent.prometheus.types.MetricFamily; +import org.hawkular.agent.prometheus.types.MetricType; +import org.hawkular.agent.prometheus.types.Summary; + +public class AssertUtils { + + public static void assertMetricFamily(MetricFamily expectedFamily, MetricFamily metricFamily) { + assertEquals(expectedFamily.getName(), metricFamily.getName()); + assertEquals(expectedFamily.getHelp(), metricFamily.getHelp()); + assertEquals(expectedFamily.getType(), metricFamily.getType()); + List expectedMetrics = expectedFamily.getMetrics(); + List metrics = metricFamily.getMetrics(); + assertEquals(expectedMetrics.size(), metrics.size()); + + for (int i = 0; i < expectedMetrics.size(); i++) { + Metric expectedMetric = expectedMetrics.get(i); + Metric metric = metrics.get(i); + assertBaseMetric(expectedMetric, metric, expectedFamily.getName()); + MetricType type = expectedFamily.getType(); + switch (type) { + case COUNTER: + assertCounter((Counter) expectedMetric, metric); + break; + case GAUGE: + case UNKNOWN: + assertGauge((Gauge) expectedMetric, metric); + break; + case SUMMARY: + assertSummary((Summary) expectedMetric, metric); + break; + default: + throw new IllegalArgumentException("Unsupported metric type: " + type); + } + } + } + + public static void assertBaseMetric(Metric expectedMetric, Metric metric, String name) { + assertEquals(name, metric.getName()); + if(expectedMetric.getLabels() != null) { + assertEquals(expectedMetric.getLabels().size(), metric.getLabels().size()); + for (Map.Entry expectedLabel : expectedMetric.getLabels().entrySet()) { + assertEquals(expectedLabel.getValue(), metric.getLabels().get(expectedLabel.getKey())); + } + } + optionalAssertEquals(expectedMetric.getTimestamp(), metric.getTimestamp()); + optionalAssertEquals(expectedMetric.getCreated(), metric.getCreated()); + } + + public static void optionalAssertEquals(Object expected, Object actual) { + if (expected != null) { + assertEquals(expected, actual); + } + } + + public static void assertCounter(Counter expectedMetric, Metric metric) { + assertEquals(Counter.class, metric.getClass()); + Counter counter = (Counter) metric; + assertEquals(expectedMetric.getValue(), counter.getValue(), 0.001); + } + + public static void assertGauge(Gauge expectedMetric, Metric metric) { + assertEquals(Gauge.class, metric.getClass()); + Gauge gauge = (Gauge) metric; + assertEquals(expectedMetric.getValue(), gauge.getValue(), 0.001); + } + + public static void assertSummary(Summary expectedMetric, Metric metric) { + assertEquals(Summary.class, metric.getClass()); + Summary summary = (Summary) metric; + long sampleCount = summary.getSampleCount(); + double sampleSum = summary.getSampleSum(); + assertEquals(sampleCount, expectedMetric.getSampleCount()); + assertEquals(sampleSum, expectedMetric.getSampleSum(), 0.001); + } +} diff --git a/features/prometheus-collector/src/test/java/org/hawkular/agent/prometheus/text/OpenMetricsTextV1Test.java b/features/prometheus-collector/src/test/java/org/hawkular/agent/prometheus/text/OpenMetricsTextV1Test.java new file mode 100644 index 000000000000..86e1e2507cb4 --- /dev/null +++ b/features/prometheus-collector/src/test/java/org/hawkular/agent/prometheus/text/OpenMetricsTextV1Test.java @@ -0,0 +1,160 @@ + /* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.hawkular.agent.prometheus.text; + +import static org.hawkular.agent.prometheus.text.AssertUtils.assertMetricFamily; +import static org.junit.Assert.assertNull; + +import java.io.IOException; +import java.io.InputStream; +import java.time.Instant; +import java.util.List; +import java.util.Map; + +import org.hawkular.agent.prometheus.types.Counter; +import org.hawkular.agent.prometheus.types.Gauge; +import org.hawkular.agent.prometheus.types.MetricFamily; +import org.hawkular.agent.prometheus.types.MetricType; +import org.hawkular.agent.prometheus.types.Summary; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + * + */ +public class OpenMetricsTextV1Test { + + private static final Logger LOGGER = LoggerFactory.getLogger(OpenMetricsTextV1Test.class); + + @Test + public void testOpenMetrics() throws IOException { + + MetricFamily gauge = new MetricFamily.Builder().setName("go_goroutines") + .setType(MetricType.GAUGE) + .setHelp("Number of goroutines that currently exist.") + .addMetric(Gauge.builder().setName("go_goroutines").setValue(69).build()) + .build(); + + + MetricFamily counter1 = new MetricFamily.Builder().setName("process_cpu_seconds") + .setType(MetricType.COUNTER) + .setHelp("Total user and system CPU time spent in seconds.") + .addMetric(Counter.builder().setName("process_cpu_seconds").setValue(4.20072246e+06).build()) + .build(); + + + Instant sampleTs1 = Instant.ofEpochSecond(1488452300L); + + MetricFamily counter2 = new MetricFamily.Builder().setName("http_requests") + .setType(MetricType.COUNTER) + .setHelp("Total number of HTTP requests made.") + .addMetric(Counter.builder().setName("http_requests") + .addLabels(Map.of("method", "get", "code", "200")) + .setValue(1027) + .setTimestamp(sampleTs1).build()) + .addMetric(Counter.builder().setName("http_requests") + .addLabels(Map.of("method", "post", "code", "200")) + .setValue(32) + .setTimestamp(sampleTs1).build()) + .build(); + + + Instant sampleTs2 = Instant.ofEpochSecond(1520879607L); + Instant createdTs = Instant.ofEpochSecond(1605281325L); + + MetricFamily counter3 = new MetricFamily.Builder().setName("foo") + .setType(MetricType.COUNTER) + .setHelp("") + .addMetric(Counter.builder().setName("foo") + .addLabel("color", "red") + .setValue(17) + .setTimestamp(sampleTs2) + .setCreated(createdTs).build()) + .addMetric(Counter.builder().setName("foo") + .addLabel("color", "green") + .setValue(10) + .setTimestamp(sampleTs2) + .setCreated(createdTs).build()) + .build(); + + + MetricFamily summary = new MetricFamily.Builder().setName("acme_http_router_request_seconds") + .setType(MetricType.SUMMARY) + .setHelp("Latency though all of ACME's HTTP request router.") + .addMetric(Summary.builder() + .setName("acme_http_router_request_seconds") + .addLabels(Map.of("path", "/api/v1", "method", "GET")) + .setSampleSum(9036.32) + .setSampleCount(807283) + .setCreated(Instant.ofEpochSecond(1605281325L)) + .build()) + .addMetric(Summary.builder() + .setName("acme_http_router_request_seconds") + .addLabels(Map.of("path", "/api/v2", "method", "POST")) + .setSampleSum(479.3) + .setSampleCount(34) + .setCreated(Instant.ofEpochSecond(1605281325L)) + .build() + ) + .build(); + + MetricFamily unknownMetric = new MetricFamily.Builder().setName("unknown_metric_1") + .setType(MetricType.UNKNOWN) + .setHelp("A metric with an unknown type") + .addMetric(Gauge.builder().setName("unknown_metric_1").setValue(42).build()) + .build(); + + MetricFamily untyped1 = new MetricFamily.Builder().setName("unknown_metric_2") + .setType(MetricType.UNKNOWN) + .setHelp("Missing type, defaults to unknown") + .addMetric(Gauge.builder().setName("unknown_metric_2").setValue(55).build()) + .build(); + + MetricFamily untyped2 = new MetricFamily.Builder().setName("unknown_metric_3") + .setType(MetricType.UNKNOWN) + .setHelp("") + .addMetric(Gauge.builder().setName("unknown_metric_3").setValue(66).build()) + .build(); + + MetricFamily anotherGauge = new MetricFamily.Builder().setName("another_gauge") + .setType(MetricType.GAUGE) + .setHelp("Another gauge metric.") + .addMetric(Gauge.builder().setName("another_gauge").setValue(128).build()) + .build(); + + List expectedMetricFamilies = List.of(gauge, counter1, counter2, counter3, summary, unknownMetric, untyped1, untyped2, anotherGauge); + + try(InputStream input = PrometheusTextV0_0_4Test.class.getResourceAsStream("/openmetrics.txt")){ + TextPrometheusMetricDataParser parser = new TextPrometheusMetricDataParser(input, OpenMetricsFormatStrategy.INSTANCE); + for (MetricFamily expected : expectedMetricFamilies) { + LOGGER.info("Validating metric {}", expected.getName()); + MetricFamily metricFamily = parser.parse(); + assertMetricFamily(expected, metricFamily); + } + assertNull("Expected no more metric families", parser.parse()); + } + } + + +} diff --git a/features/prometheus-collector/src/test/java/org/hawkular/agent/prometheus/text/PrometheusTextV0_0_4Test.java b/features/prometheus-collector/src/test/java/org/hawkular/agent/prometheus/text/PrometheusTextV0_0_4Test.java new file mode 100644 index 000000000000..a149e183c85a --- /dev/null +++ b/features/prometheus-collector/src/test/java/org/hawkular/agent/prometheus/text/PrometheusTextV0_0_4Test.java @@ -0,0 +1,108 @@ + /* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.hawkular.agent.prometheus.text; + +import static org.hawkular.agent.prometheus.text.AssertUtils.assertMetricFamily; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; + +import org.hawkular.agent.prometheus.types.Counter; +import org.hawkular.agent.prometheus.types.MetricFamily; +import org.hawkular.agent.prometheus.types.MetricType; +import org.junit.Test; + +/** + * Validates PrometheusText0.0.4 format + * + */ +public class PrometheusTextV0_0_4Test { + + @Test + public void testSingleCounter() throws IOException { + MetricFamily counter1 = new MetricFamily.Builder().setName("http_requests_total") + .setType(MetricType.COUNTER) + .setHelp("Total number of HTTP requests made.") + .addMetric(Counter.builder().setName("http_requests_total") + .addLabels(Map.of("code", "200", "handler", "prometheus", "method", "get")) + .setValue(162030) + .build()) + .addMetric(Counter.builder().setName("http_requests_total") + .addLabels(Map.of("code", "200", "handler", "query", "method", "get")) + .setValue(40) + .build()) + .addMetric(Counter.builder().setName("http_requests_total") + .addLabels(Map.of("code", "200", "handler", "series", "method", "get")) + .setValue(Double.NaN) + .build()) + .addMetric(Counter.builder().setName("http_requests_total") + .addLabels(Map.of("code", "400", "handler", "query", "method", "get")) + .setValue(Double.POSITIVE_INFINITY) + .build()) + .addMetric(Counter.builder().setName("http_requests_total") + .addLabels(Map.of("code", "400", "handler", "series", "method", "get")) + .setValue(Double.NEGATIVE_INFINITY) + .build()) + .build(); + + try (InputStream input = PrometheusTextV0_0_4Test.class.getResourceAsStream("/prometheus-counter.txt")) { + TextPrometheusMetricDataParser parser = new TextPrometheusMetricDataParser(input, PrometheusTextFormatStrategy.INSTANCE); + MetricFamily metricFamily = parser.parse(); + assertMetricFamily(counter1, metricFamily); + } + } + + @Test + public void testThreeCounters() throws IOException { + MetricFamily counter1 = new MetricFamily.Builder().setName("one_counter_total") + .setType(MetricType.COUNTER) + .setHelp("This is the first") + .addMetric(Counter.builder().setName("one_counter_total").setValue(111).build()) + .build(); + + MetricFamily counter2 = new MetricFamily.Builder().setName("two_counter_total") + .setType(MetricType.COUNTER) + .setHelp("This is the second") + .addMetric(Counter.builder().setName("two_counter_total").setValue(222).build()) + .build(); + + MetricFamily counter3 = new MetricFamily.Builder().setName("three_counter_total") + .setType(MetricType.COUNTER) + .setHelp("This is the third with type specified first") + .addMetric(Counter.builder().setName("three_counter_total").setValue(333).build()) + .build(); + + try (InputStream input = PrometheusTextV0_0_4Test.class.getResourceAsStream("/prometheus-three-counters.txt")) { + TextPrometheusMetricDataParser parser = new TextPrometheusMetricDataParser(input, PrometheusTextFormatStrategy.INSTANCE); + + MetricFamily metricFamily1 = parser.parse(); + assertMetricFamily(counter1, metricFamily1); + + MetricFamily metricFamily2 = parser.parse(); + assertMetricFamily(counter2, metricFamily2); + + MetricFamily metricFamily3 = parser.parse(); + assertMetricFamily(counter3, metricFamily3); + } + } +} diff --git a/features/prometheus-collector/src/test/java/org/opennnms/netmgt/collectd/prometheus/PrometheusCollectorIT.java b/features/prometheus-collector/src/test/java/org/opennnms/netmgt/collectd/prometheus/PrometheusCollectorIT.java index 14a8fef83d5a..d67618b340d2 100644 --- a/features/prometheus-collector/src/test/java/org/opennnms/netmgt/collectd/prometheus/PrometheusCollectorIT.java +++ b/features/prometheus-collector/src/test/java/org/opennnms/netmgt/collectd/prometheus/PrometheusCollectorIT.java @@ -25,6 +25,7 @@ import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static java.util.Collections.singletonList; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; @@ -92,12 +93,20 @@ public class PrometheusCollectorIT { public void setUp() { stubFor(get(urlEqualTo("/metrics")) .willReturn(aResponse() - .withHeader("Content-Type", "Content-Type: text/plain; version=0.0.4") + .withHeader("Content-Type", "text/plain; version=0.0.4") .withBodyFile("linux.metrics"))); stubFor(get(urlEqualTo("/nephron_on_flink")) .willReturn(aResponse() - .withHeader("Content-Type", "Content-Type: text/plain; version=0.0.4") + .withHeader("Content-Type", "text/plain; version=0.0.4") .withBodyFile("flink.metrics"))); + stubFor(get(urlEqualTo("/actuator/prometheus")) + .willReturn(aResponse() + .withHeader("Content-Type", "application/openmetrics-text;version=1.0.0") + .withBodyFile("open.metrics"))); + stubFor(get(urlEqualTo("/open_noncompliant")) + .willReturn(aResponse() + .withHeader("Content-Type", "application/openmetrics-text") + .withBodyFile("open_noncompliant.metrics"))); } @Test @@ -419,6 +428,77 @@ public void canGatherCpuMetrics() { "0/nodeExporterCPU/cpu7/node-exporter-cpu/system[null,200.86]", "0/nodeExporterCPU/cpu7/node-exporter-cpu/user[null,1203.71]"), collectionSetKeys); } + + @Test + public void canGatherOpenMetricsGauges() { + Collection collection = new Collection(); + + Group executorPool = new Group(); + executorPool.setName("executor-pool"); + executorPool.setFilterExp("name matches 'executor_pool_size_threads'"); + executorPool.setResourceType("node"); + + NumericAttribute executorPoolSize = new NumericAttribute(); + executorPoolSize.setAliasExp("labels[name]"); + executorPool.getNumericAttribute().add(executorPoolSize); + + // Collect! + CollectionSet collectionSet = collectOpenMetrics(collection,executorPool); + + // Verify + List collectionSetKeys = CollectionSetUtils.flatten(collectionSet); + assertThat(collectionSetKeys, hasSize(1)); + assertThat(collectionSetKeys, hasItem("0/executor-pool/taskExecutor[null,0.0]")); + } + + @Test + public void canGatherOpenMetricsCounters() { + Collection collection = new Collection(); + + Group logbackEvents = new Group(); + logbackEvents.setName("logback-events"); + logbackEvents.setFilterExp("name matches 'logback_events'"); + logbackEvents.setGroupByExp("labels[level]"); + logbackEvents.setResourceType("node"); + + NumericAttribute logbackEventCount = new NumericAttribute(); + logbackEventCount.setAliasExp("labels[level]"); + logbackEvents.getNumericAttribute().add(logbackEventCount); + + // Collect! + CollectionSet collectionSet = collectOpenMetrics(collection,logbackEvents); + + // Verify + List collectionSetKeys = CollectionSetUtils.flatten(collectionSet); + assertThat(collectionSetKeys, hasSize(5)); + assertThat(collectionSetKeys, hasItem("0/logback-events/debug[null,2.0]")); + assertThat(collectionSetKeys, hasItem("0/logback-events/error[null,9170.0]")); + assertThat(collectionSetKeys, hasItem("0/logback-events/info[null,251.0]")); + assertThat(collectionSetKeys, hasItem("0/logback-events/trace[null,1.0]")); + assertThat(collectionSetKeys, hasItem("0/logback-events/warn[null,2876.0]")); + } + + @Test + public void canGatherOpenMetricsNonCompliant() { + Collection collection = new Collection(); + + Group executorPool = new Group(); + executorPool.setName("executor-pool"); + executorPool.setFilterExp("name matches 'executor_pool_size_threads'"); + executorPool.setResourceType("node"); + + NumericAttribute executorPoolSize = new NumericAttribute(); + executorPoolSize.setAliasExp("labels[name]"); + executorPool.getNumericAttribute().add(executorPoolSize); + + // Collect! + CollectionSet collectionSet = collect(collection,singletonList(executorPool), "open_noncompliant"); + + // Verify + List collectionSetKeys = CollectionSetUtils.flatten(collectionSet); + assertThat(collectionSetKeys, hasSize(1)); + assertThat(collectionSetKeys, hasItem("0/executor-pool/taskExecutor[null,0.0]")); + } private CollectionSet collect(Collection collection, List groups) { return collect(collection, groups, "metrics"); @@ -427,6 +507,10 @@ private CollectionSet collect(Collection collection, List groups) { private CollectionSet collectNephronMetrics(Collection collection, Group... groups) { return collect(collection, Arrays.asList(groups), "nephron_on_flink"); } + + private CollectionSet collectOpenMetrics(Collection collection, Group... groups) { + return collect(collection, Arrays.asList(groups), "actuator/prometheus"); + } private CollectionSet collect(Collection collection, List groups, String path) { // Create the agent diff --git a/features/prometheus-collector/src/test/resources/__files/open.metrics b/features/prometheus-collector/src/test/resources/__files/open.metrics new file mode 100644 index 000000000000..b10f1a3b1b22 --- /dev/null +++ b/features/prometheus-collector/src/test/resources/__files/open.metrics @@ -0,0 +1,14 @@ +# TYPE executor_pool_size_threads gauge +# HELP executor_pool_size_threads The current number of threads in the pool +executor_pool_size_threads{name="taskExecutor"} 0.0 +# TYPE jvm_classes_unloaded_classes counter +# HELP jvm_classes_unloaded_classes The total number of classes unloaded since the Java virtual machine has started execution +jvm_classes_unloaded_classes_total 1.0 +# TYPE logback_events counter +# HELP logback_events Number of log events that were enabled by the effective log level +logback_events_total{level="warn"} 2876.0 +logback_events_total{level="debug"} 2.0 +logback_events_total{level="error"} 9170.0 +logback_events_total{level="trace"} 1.0 +logback_events_total{level="info"} 251.0 +# EOF \ No newline at end of file diff --git a/features/prometheus-collector/src/test/resources/__files/open_noncompliant.metrics b/features/prometheus-collector/src/test/resources/__files/open_noncompliant.metrics new file mode 100644 index 000000000000..a3a3d94c0193 --- /dev/null +++ b/features/prometheus-collector/src/test/resources/__files/open_noncompliant.metrics @@ -0,0 +1,4 @@ +# TYPE executor_pool_size_threads +# HELP executor_pool_size_threads Missing the type, assumes unknown/gauge +executor_pool_size_threads{name="taskExecutor"} 0.0 +# EOF \ No newline at end of file diff --git a/features/prometheus-collector/src/test/resources/mocklogger.properties b/features/prometheus-collector/src/test/resources/mocklogger.properties new file mode 100644 index 000000000000..3e6bfd71f35f --- /dev/null +++ b/features/prometheus-collector/src/test/resources/mocklogger.properties @@ -0,0 +1,2 @@ +org.opennms.core.test.mockLogger.defaultLogLevel=debug +org.opennms.core.test.mockLogger.log.org=debug \ No newline at end of file diff --git a/features/prometheus-collector/src/test/resources/openmetrics.txt b/features/prometheus-collector/src/test/resources/openmetrics.txt new file mode 100644 index 000000000000..0a93ccf5f58d --- /dev/null +++ b/features/prometheus-collector/src/test/resources/openmetrics.txt @@ -0,0 +1,43 @@ +# TYPE go_goroutines gauge +# HELP go_goroutines Number of goroutines that currently exist. +go_goroutines 69 +# TYPE process_cpu_seconds counter +# UNIT process_cpu_seconds seconds +# HELP process_cpu_seconds Total user and system CPU time spent in seconds. +process_cpu_seconds_total 4.20072246e+06 +# TYPE http_requests counter +# UNIT http_requests requests +# HELP http_requests Total number of HTTP requests made. +http_requests_total{method="get",code="200"} 1027 1.488452300E9 +http_requests_total{method="post",code="200"} 32 1.488452300E9 +# TYPE foo counter +foo_total{color="red"} 17.0 1520879607.789 +foo_created{color="red"} 1605281325.0 +foo_total{color="green"} 10 1520879607.789 +foo_created{color="green"} 1605281325.0 +# TYPE acme_http_router_request_seconds summary +# UNIT acme_http_router_request_seconds seconds +# HELP acme_http_router_request_seconds Latency though all of ACME's HTTP request router. +acme_http_router_request_seconds_sum{path="/api/v1",method="GET"} 9036.32 +acme_http_router_request_seconds_count{path="/api/v1",method="GET"} 807283.0 +acme_http_router_request_seconds_created{path="/api/v1",method="GET"} 1605281325.0 +acme_http_router_request_seconds_sum{path="/api/v2",method="POST"} 479.3 +acme_http_router_request_seconds_count{path="/api/v2",method="POST"} 34.0 +acme_http_router_request_seconds_created{path="/api/v2",method="POST"} 1605281325.0 +# TYPE unknown_metric_1 unknown +# HELP unknown_metric_1 A metric with an unknown type +unknown_metric_1 42 +unknown_metric_1_garbage 99 +garbage 12 +more_garbage 3 +# HELP unknown_metric_2 Missing type, defaults to unknown +unknown_metric_2 55 +# TYPE unknown_metric_3 +unknown_metric_3 66 +# HELP acme_count A metric with an invalid type. Should be ignored +# TYPE acme_count invalid_type +acme_count 10 +# TYPE another_gauge gauge +# HELP another_gauge Another gauge metric. +another_gauge 128 +# EOF \ No newline at end of file