From ea7610b84fbf01f1181dea195322e45d69a93c1d Mon Sep 17 00:00:00 2001 From: Dino Date: Wed, 8 Jul 2026 01:33:16 -0500 Subject: [PATCH 01/15] First pass at a PostgresFlowRepository and PostgresFlowQueryService --- .../features/src/main/resources/features.xml | 14 + features/flows/pom.xml | 1 + features/flows/postgres/pom.xml | 144 ++++ .../flows/postgres/BatchingFlowWriter.java | 164 +++++ .../netmgt/flows/postgres/FlowProration.java | 121 +++ .../netmgt/flows/postgres/FlowRow.java | 50 ++ .../netmgt/flows/postgres/FlowRowMapper.java | 153 ++++ .../postgres/PostgresFlowQueryService.java | 696 ++++++++++++++++++ .../postgres/PostgresFlowRepository.java | 181 +++++ .../OSGI-INF/blueprint/blueprint.xml | 61 ++ .../netmgt/flows/postgres/changelog.xml | 120 +++ .../postgres/PostgresFlowRepositoryIT.java | 295 ++++++++ .../postgres/ProportionalSumParityTest.java | 245 ++++++ .../filtered/etc/vacuumd-configuration.xml | 8 + opennms-full-assembly/pom.xml | 1 + 15 files changed, 2254 insertions(+) create mode 100644 features/flows/postgres/pom.xml create mode 100644 features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/BatchingFlowWriter.java create mode 100644 features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowProration.java create mode 100644 features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowRow.java create mode 100644 features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowRowMapper.java create mode 100644 features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresFlowQueryService.java create mode 100644 features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepository.java create mode 100644 features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml create mode 100644 features/flows/postgres/src/main/resources/org/opennms/netmgt/flows/postgres/changelog.xml create mode 100644 features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepositoryIT.java create mode 100644 features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/ProportionalSumParityTest.java diff --git a/container/features/src/main/resources/features.xml b/container/features/src/main/resources/features.xml index f15ace5a8691..cae2a192d9ad 100644 --- a/container/features/src/main/resources/features.xml +++ b/container/features/src/main/resources/features.xml @@ -1214,6 +1214,20 @@ mvn:org.opennms.features.flows.classification/org.opennms.features.flows.classification.shell/${project.version} mvn:org.opennms.features.telemetry.protocols.netflow.xml/org.opennms.features.telemetry.protocols.netflow.xml.core/${project.version} + +
Optional PostgreSQL-backed FlowRepository/FlowQueryService (jsonb). Install alongside or instead of the Elasticsearch flow persistence.
+ opennms-flows + spring-jdbc + postgresql + mvn:com.fasterxml.jackson.core/jackson-databind/${jackson2Version} + + mvn:org.yaml/snakeyaml/1.31 + mvn:org.liquibase/liquibase-core/${liquibaseVersion} + mvn:org.opennms.features.flows/org.opennms.features.flows.postgres/${project.version} + opennms-health opennms-situation-feedback-api diff --git a/features/flows/pom.xml b/features/flows/pom.xml index 31a342480a4f..99edafec0914 100644 --- a/features/flows/pom.xml +++ b/features/flows/pom.xml @@ -15,6 +15,7 @@ processing classification elastic + postgres itests rest kafka-persistence diff --git a/features/flows/postgres/pom.xml b/features/flows/postgres/pom.xml new file mode 100644 index 000000000000..abbaadd70236 --- /dev/null +++ b/features/flows/postgres/pom.xml @@ -0,0 +1,144 @@ + + + + org.opennms.features + org.opennms.features.flows + 37.0.0-SNAPSHOT + + 4.0.0 + org.opennms.features.flows + org.opennms.features.flows.postgres + bundle + OpenNMS :: Features :: Flows :: Postgres + Prototype FlowRepository/FlowQueryService backed by PostgreSQL (jsonb). + + true + + + + + org.apache.felix + maven-bundle-plugin + true + + + ${project.artifactId} + ${project.version} + + * + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + + + + org.osgi + osgi.core + provided + + + org.osgi + osgi.cmpn + provided + + + org.slf4j + slf4j-api + provided + + + + + org.opennms.features.flows + org.opennms.features.flows.processing + ${project.version} + + + + org.opennms.features.flows + org.opennms.features.flows.api + ${project.version} + + + + org.opennms + opennms-dao-api + + + + org.opennms.core + org.opennms.core.db + ${project.version} + + + + org.apache.servicemix.bundles + org.apache.servicemix.bundles.spring-jdbc + + + com.fasterxml.jackson.core + jackson-databind + + + com.google.guava + guava + ${guavaVersion} + + + io.dropwizard.metrics + metrics-core + ${dropwizardMetricsVersion} + + + org.liquibase + liquibase-core + ${liquibaseVersion} + + + + junit + junit + test + + + org.hamcrest + hamcrest-library + test + + + org.mockito + mockito-core + test + + + + org.testcontainers + testcontainers + ${testcontainers.version} + test + + + org.testcontainers + postgresql + ${testcontainers.version} + test + + + org.postgresql + postgresql + test + + + diff --git a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/BatchingFlowWriter.java b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/BatchingFlowWriter.java new file mode 100644 index 000000000000..635357568042 --- /dev/null +++ b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/BatchingFlowWriter.java @@ -0,0 +1,164 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.postgres; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; + +/** + * Buffers items into batches that are flushed on whichever condition fires first: + * a document-count threshold ({@code batchSize}) or a time threshold ({@code flushIntervalMs}). + * + *

Backpressure policy is drop-newest: {@link #add(Object)} performs a non-blocking + * offer onto a bounded queue and, when the queue is full (the sink cannot keep up), increments a + * {@code dropped} meter and returns without blocking. This protects the upstream flow pipeline / + * parser threads from stalling when PostgreSQL falls behind. + * + * @param the buffered item type (a mapped flow row) + */ +public class BatchingFlowWriter implements AutoCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(BatchingFlowWriter.class); + + private final BlockingQueue queue; + private final int batchSize; + private final long flushIntervalMs; + private final Consumer> flush; + + private final Meter enqueued; + private final Meter dropped; + private final Meter flushed; + private final Timer flushTimer; + + private volatile boolean running = false; + private Thread worker; + + public BatchingFlowWriter(final String name, + final int queueCapacity, + final int batchSize, + final long flushIntervalMs, + final Consumer> flush, + final MetricRegistry metrics) { + if (queueCapacity < 1) { + throw new IllegalArgumentException("queueCapacity must be >= 1"); + } + if (batchSize < 1) { + throw new IllegalArgumentException("batchSize must be >= 1"); + } + this.queue = new ArrayBlockingQueue<>(queueCapacity); + this.batchSize = batchSize; + this.flushIntervalMs = flushIntervalMs > 0 ? flushIntervalMs : 500; + this.flush = flush; + this.enqueued = metrics.meter(MetricRegistry.name(name, "enqueued")); + this.dropped = metrics.meter(MetricRegistry.name(name, "dropped")); + this.flushed = metrics.meter(MetricRegistry.name(name, "flushed")); + this.flushTimer = metrics.timer(MetricRegistry.name(name, "flush")); + metrics.register(MetricRegistry.name(name, "queueSize"), (com.codahale.metrics.Gauge) queue::size); + } + + public synchronized void start() { + if (running) { + return; + } + running = true; + worker = new Thread(this::drainLoop, "postgres-flow-writer"); + worker.setDaemon(true); + worker.start(); + } + + /** + * Enqueue an item for batched persistence. Non-blocking; drops the item (and counts it) when + * the bounded queue is full. + */ + public void add(final T item) { + if (queue.offer(item)) { + enqueued.mark(); + } else { + dropped.mark(); + } + } + + private void drainLoop() { + final List batch = new ArrayList<>(batchSize); + long deadline = System.currentTimeMillis() + flushIntervalMs; + while (running || !queue.isEmpty()) { + try { + final long wait = Math.max(0, deadline - System.currentTimeMillis()); + final T item = queue.poll(wait, TimeUnit.MILLISECONDS); + if (item != null) { + batch.add(item); + if (batch.size() >= batchSize) { // count-based flush + doFlush(batch); + deadline = System.currentTimeMillis() + flushIntervalMs; + } + } else if (!batch.isEmpty()) { // time-based flush + doFlush(batch); + deadline = System.currentTimeMillis() + flushIntervalMs; + } else { + deadline = System.currentTimeMillis() + flushIntervalMs; + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + if (!batch.isEmpty()) { + doFlush(batch); + } + } + + private void doFlush(final List batch) { + try (Timer.Context ignored = flushTimer.time()) { + flush.accept(new ArrayList<>(batch)); + flushed.mark(batch.size()); + } catch (final Exception e) { + LOG.warn("Failed to flush a batch of {} flow rows; the batch is dropped.", batch.size(), e); + } finally { + batch.clear(); + } + } + + @Override + public synchronized void close() { + running = false; + if (worker != null) { + worker.interrupt(); + try { + worker.join(TimeUnit.SECONDS.toMillis(30)); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } + worker = null; + } + } +} \ No newline at end of file diff --git a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowProration.java b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowProration.java new file mode 100644 index 000000000000..d11418c1ceaa --- /dev/null +++ b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowProration.java @@ -0,0 +1,121 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.postgres; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Pure-Java reference for the byte-proration arithmetic that {@link PostgresFlowQueryService} + * renders as SQL. It is a line-for-line mirror of the {@code proratedForWindow} / + * {@code proratedForBucket} / {@code SAMPLING} expressions and exists so the proportional-sum + * behaviour can be unit-tested (against the Elasticsearch drift-plugin's {@code ProportionalSumAggregator}) + * without a live PostgreSQL. Changing the SQL means changing this in lockstep. + * + *

A flow's bytes (scaled by its sampling interval) are distributed across time buckets in + * proportion to the overlap of {@code [delta_switched, last_switched]} with each bucket, using the + * full flow duration as the denominator (a flow straddling the window contributes only its + * in-window fraction), with a zero-duration flow contributing its whole value to the single bucket + * containing its instant. + */ +final class FlowProration { + + private FlowProration() { + } + + /** Mirrors the {@code SAMPLING} SQL CASE: scale only by a finite, non-zero interval; else 1. */ + static double effectiveSampling(final Double samplingInterval) { + if (samplingInterval == null || !Double.isFinite(samplingInterval) || samplingInterval == 0.0) { + return 1.0; + } + return samplingInterval; + } + + /** + * Prorated value for the single summary bucket {@code [s, e)} (half-open at {@code e}). + * Mirrors {@code proratedForWindow(s, e)}. + */ + static double summaryValue(final long bytes, final Double samplingInterval, + final long delta, final long last, final long s, final long e) { + final double value = bytes * effectiveSampling(samplingInterval); + if (last <= delta) { + return (delta >= s && delta < e) ? value : 0.0; + } + final double overlap = Math.max(0, Math.min(last, e) - Math.max(delta, s)); + return value * overlap / (double) (last - delta); + } + + /** Bucket starts a series row expands to, epoch-aligned (origin 0 — Elastic's only mode). */ + static long[] seriesBucketStarts(final long delta, final long last, final long s, final long e, final long step) { + return seriesBucketStarts(delta, last, s, e, step, 0L); + } + + /** + * Bucket starts a series row expands to, aligned to {@code origin} + k*{@code step}. Mirrors the + * {@code generate_series(...)} bounds; floor toward -inf matches PostgreSQL FLOOR and the drift + * plugin's {@code round(v - offset) + offset}. + */ + static long[] seriesBucketStarts(final long delta, final long last, final long s, final long e, + final long step, final long origin) { + final long lower = Math.floorDiv(Math.max(delta, s) - origin, step) * step + origin; + final long upper = Math.min(last, e); + if (upper < lower) { + return new long[0]; + } + final int n = (int) ((upper - lower) / step) + 1; + final long[] out = new long[n]; + long b = lower; + for (int i = 0; i < n; i++) { + out[i] = b; + b += step; + } + return out; + } + + /** Contribution of a row to a single series bucket {@code [bucketStart, bucketStart+step)}. Mirrors {@code proratedForBucket(step)}. */ + static double seriesContribution(final long bytes, final Double samplingInterval, + final long delta, final long last, final long bucketStart, final long step) { + final double value = bytes * effectiveSampling(samplingInterval); + if (last <= delta) { + return value; + } + final double overlap = Math.max(0, Math.min(last, bucketStart + step) - Math.max(delta, bucketStart)); + return value * overlap / (double) (last - delta); + } + + /** Full series (bucketStart -> value) a single row contributes over the window {@code [s, e]} at {@code step}, origin 0. */ + static Map series(final long bytes, final Double samplingInterval, + final long delta, final long last, final long s, final long e, final long step) { + return series(bytes, samplingInterval, delta, last, s, e, step, 0L); + } + + /** Full series (bucketStart -> value) a single row contributes over {@code [s, e]} at {@code step}, buckets aligned to {@code origin}. */ + static Map series(final long bytes, final Double samplingInterval, + final long delta, final long last, final long s, final long e, + final long step, final long origin) { + final Map out = new LinkedHashMap<>(); + for (final long bucketStart : seriesBucketStarts(delta, last, s, e, step, origin)) { + out.put(bucketStart, seriesContribution(bytes, samplingInterval, delta, last, bucketStart, step)); + } + return out; + } +} \ No newline at end of file diff --git a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowRow.java b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowRow.java new file mode 100644 index 000000000000..6311eff06112 --- /dev/null +++ b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowRow.java @@ -0,0 +1,50 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.postgres; + +import java.sql.Timestamp; + +/** + * A single row destined for the {@code flow} table: the promoted "hot" columns used by the + * query/filter paths, plus the full enriched flow serialized as a jsonb document. + */ +public class FlowRow { + public Timestamp flowTs; // @timestamp equivalent; partition key + TimeRangeFilter selection + public Long deltaSwitched; // netflow.delta_switched (proration range start, epoch ms) + public Long lastSwitched; // netflow.last_switched (proration range end, epoch ms) + public Long firstSwitched; + public Long bytes; + public Long packets; + public Double samplingInterval; + public String direction; // INGRESS / EGRESS + public String application; + public String convoKey; + public String srcAddr; + public String dstAddr; + public Integer protocol; + public Integer dscp; + public Integer exporterNodeId; + public Integer inputSnmp; + public Integer outputSnmp; + public String location; + public String documentJson; // full enriched flow as JSON (stored as jsonb) +} \ No newline at end of file diff --git a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowRowMapper.java b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowRowMapper.java new file mode 100644 index 000000000000..d9ef1ae14a5a --- /dev/null +++ b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowRowMapper.java @@ -0,0 +1,153 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.postgres; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.opennms.integration.api.v1.flows.Flow; + +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Maps an enriched {@link Flow} to a {@link FlowRow}: promoted hot columns for querying, plus a + * jsonb {@code document} whose keys mirror the Elasticsearch {@code netflow.*} field names so that + * field-based queries ({@code LimitedCardinalityField}) and ad-hoc jsonb access use the same names. + */ +public class FlowRowMapper { + + private final ObjectMapper objectMapper; + + public FlowRowMapper(final ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + public FlowRow toRow(final Flow flow) { + final FlowRow r = new FlowRow(); + r.flowTs = toTs(flow.getTimestamp()); + r.deltaSwitched = toEpochMilli(flow.getDeltaSwitched()); + r.lastSwitched = toEpochMilli(flow.getLastSwitched()); + r.firstSwitched = toEpochMilli(flow.getFirstSwitched()); + r.bytes = flow.getBytes(); + r.packets = flow.getPackets(); + r.samplingInterval = flow.getSamplingInterval(); + r.direction = flow.getDirection() != null ? flow.getDirection().name() : null; + r.application = flow.getApplication(); + r.convoKey = flow.getConvoKey(); + r.srcAddr = flow.getSrcAddr(); + r.dstAddr = flow.getDstAddr(); + r.protocol = flow.getProtocol(); + r.dscp = flow.getDscp(); + r.inputSnmp = flow.getInputSnmp(); + r.outputSnmp = flow.getOutputSnmp(); + r.location = flow.getLocation(); + if (flow.getExporterNodeInfo() != null) { + r.exporterNodeId = flow.getExporterNodeInfo().getNodeId(); + } + r.documentJson = toDocumentJson(flow); + return r; + } + + /** + * Full-fidelity document keyed on the same names the Elastic {@code FlowDocument} uses, so no + * field is lost and jsonb access matches the documented schema. Null values are omitted. + */ + String toDocumentJson(final Flow flow) { + final Map doc = new LinkedHashMap<>(); + put(doc, "@timestamp", toEpochMilli(flow.getTimestamp())); + put(doc, "location", flow.getLocation()); + put(doc, "host", flow.getHost()); + put(doc, "netflow.application", flow.getApplication()); + put(doc, "netflow.convo_key", flow.getConvoKey()); + put(doc, "netflow.bytes", flow.getBytes()); + put(doc, "netflow.packets", flow.getPackets()); + put(doc, "netflow.direction", flow.getDirection()); + put(doc, "netflow.first_switched", toEpochMilli(flow.getFirstSwitched())); + put(doc, "netflow.delta_switched", toEpochMilli(flow.getDeltaSwitched())); + put(doc, "netflow.last_switched", toEpochMilli(flow.getLastSwitched())); + put(doc, "netflow.sampling_interval", flow.getSamplingInterval()); + put(doc, "netflow.sampling_algorithm", flow.getSamplingAlgorithm()); + put(doc, "netflow.src_addr", flow.getSrcAddr()); + put(doc, "netflow.src_addr_hostname", flow.getSrcAddrHostname().orElse(null)); + put(doc, "netflow.src_port", flow.getSrcPort()); + put(doc, "netflow.src_as", flow.getSrcAs()); + put(doc, "netflow.src_mask_len", flow.getSrcMaskLen()); + put(doc, "netflow.dst_addr", flow.getDstAddr()); + put(doc, "netflow.dst_addr_hostname", flow.getDstAddrHostname().orElse(null)); + put(doc, "netflow.dst_port", flow.getDstPort()); + put(doc, "netflow.dst_as", flow.getDstAs()); + put(doc, "netflow.dst_mask_len", flow.getDstMaskLen()); + put(doc, "netflow.next_hop", flow.getNextHop()); + put(doc, "netflow.protocol", flow.getProtocol()); + put(doc, "netflow.ip_protocol_version", flow.getIpProtocolVersion()); + put(doc, "netflow.tos", flow.getTos()); + put(doc, "netflow.dscp", flow.getDscp()); + put(doc, "netflow.ecn", flow.getEcn()); + put(doc, "netflow.tcp_flags", flow.getTcpFlags()); + put(doc, "netflow.vlan", flow.getVlan()); + put(doc, "netflow.engine_id", flow.getEngineId()); + put(doc, "netflow.engine_type", flow.getEngineType()); + put(doc, "netflow.input_snmp", flow.getInputSnmp()); + put(doc, "netflow.output_snmp", flow.getOutputSnmp()); + put(doc, "netflow.version", flow.getNetflowVersion()); + put(doc, "netflow.src_locality", flow.getSrcLocality()); + put(doc, "netflow.dst_locality", flow.getDstLocality()); + put(doc, "netflow.flow_locality", flow.getFlowLocality()); + putNode(doc, "node_src", flow.getSrcNodeInfo()); + putNode(doc, "node_dst", flow.getDstNodeInfo()); + putNode(doc, "node_exporter", flow.getExporterNodeInfo()); + try { + return objectMapper.writeValueAsString(doc); + } catch (final Exception e) { + throw new IllegalStateException("Failed to serialize flow document to JSON", e); + } + } + + private static void put(final Map doc, final String key, final Object value) { + if (value != null) { + doc.put(key, value); + } + } + + private static void putNode(final Map doc, final String key, final Flow.NodeInfo node) { + if (node == null) { + return; + } + final Map n = new LinkedHashMap<>(); + n.put("node_id", node.getNodeId()); + n.put("interface_id", node.getInterfaceId()); + put(n, "foreign_source", node.getForeignSource()); + put(n, "foreign_id", node.getForeignId()); + put(n, "categories", node.getCategories()); + doc.put(key, n); + } + + private static Long toEpochMilli(final Instant instant) { + return instant != null ? instant.toEpochMilli() : null; + } + + private static Timestamp toTs(final Instant instant) { + return instant != null ? Timestamp.from(instant) : null; + } +} \ No newline at end of file diff --git a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresFlowQueryService.java b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresFlowQueryService.java new file mode 100644 index 000000000000..4b5abe81b712 --- /dev/null +++ b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresFlowQueryService.java @@ -0,0 +1,696 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.postgres; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.Supplier; + +import javax.sql.DataSource; + +import org.opennms.core.db.DataSourceFactory; +import org.opennms.netmgt.flows.api.Conversation; +import org.opennms.netmgt.flows.api.ConversationKey; +import org.opennms.netmgt.flows.api.Directional; +import org.opennms.netmgt.flows.api.FlowQueryService; +import org.opennms.netmgt.flows.api.Host; +import org.opennms.netmgt.flows.api.LimitedCardinalityField; +import org.opennms.netmgt.flows.api.TrafficSummary; +import org.opennms.netmgt.flows.filter.api.DscpFilter; +import org.opennms.netmgt.flows.filter.api.ExporterNodeFilter; +import org.opennms.netmgt.flows.filter.api.Filter; +import org.opennms.netmgt.flows.filter.api.SnmpInterfaceIdFilter; +import org.opennms.netmgt.flows.filter.api.TimeRangeFilter; +import org.opennms.netmgt.flows.processing.ConversationKeyUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.jdbc.core.JdbcTemplate; + +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.Table; + +/** + * Prototype {@link FlowQueryService} backed by the PostgreSQL {@code flow} table. + * + *

Replicates the Elasticsearch {@code proportional_sum} aggregation in SQL: a document's bytes + * (scaled by its sampling interval) are distributed across the query window / time-series buckets + * in proportion to the temporal overlap of {@code [delta_switched, last_switched]}, with the + * full flow duration as the denominator (so a flow straddling the window contributes only + * its in-window fraction) and a zero-duration flow contributing its whole value to a single bucket. + * + *

Not tuned for scale (functional parity first): generate_series bucket expansion, no COPY. + */ +public class PostgresFlowQueryService implements FlowQueryService { + + private static final Logger LOG = LoggerFactory.getLogger(PostgresFlowQueryService.class); + + private static final String OTHER = "Other"; + + // The two proration expressions below are the SQL rendering of FlowProration (pure-Java mirror), + // which the proportional-sum parity test checks against the drift-plugin ProportionalSumAggregator. + // Keep the three in sync. + + /** + * bytes * effectiveSampling, distributed by overlap of [delta_switched,last_switched] with the + * single summary bucket [{s},{e}) (half-open at {e}, matching the drift plugin's histogram bucket). + * The sampling factor is applied only when finite and non-zero (0/NaN/±Inf -> x1). + */ + private static String proratedForWindow(final long s, final long e) { + return "(COALESCE(f.bytes,0) * " + SAMPLING + ") * " + + "CASE WHEN f.last_switched <= f.delta_switched " + + " THEN (CASE WHEN f.delta_switched >= " + s + " AND f.delta_switched < " + e + " THEN 1.0 ELSE 0.0 END) " + + "ELSE GREATEST(0, LEAST(f.last_switched, " + e + ") - GREATEST(f.delta_switched, " + s + "))::float8 " + + "/ (f.last_switched - f.delta_switched) END"; + } + + /** Per-(row,bucket) contribution for a series with the given step, bucket = [b.bucket_ms, b.bucket_ms+step). */ + private static String proratedForBucket(final long step) { + return "(COALESCE(f.bytes,0) * " + SAMPLING + ") * " + + "CASE WHEN f.last_switched <= f.delta_switched THEN 1.0 " + + "ELSE GREATEST(0, LEAST(f.last_switched, b.bucket_ms + " + step + ") - GREATEST(f.delta_switched, b.bucket_ms))::float8 " + + "/ (f.last_switched - f.delta_switched) END"; + } + + /** Sampling multiplier: the stored interval when finite and non-zero, otherwise 1 (no scaling). */ + private static final String SAMPLING = + "(CASE WHEN f.sampling_interval IS NULL OR f.sampling_interval = 0 " + + "OR f.sampling_interval <> f.sampling_interval " + // NaN + "OR f.sampling_interval = 'Infinity'::float8 OR f.sampling_interval = '-Infinity'::float8 " + + "THEN 1 ELSE f.sampling_interval END)"; + + private String dataSourceName = "opennms"; + private int threads = 4; + + private DataSource dataSource; + private JdbcTemplate jdbcTemplate; + private ExecutorService executor; + + public void start() { + final DataSource ds; + if (this.dataSource != null) { + ds = this.dataSource; + } else { + DataSourceFactory.init(dataSourceName); + ds = DataSourceFactory.getInstance(dataSourceName); + } + this.jdbcTemplate = new JdbcTemplate(ds); + this.executor = Executors.newFixedThreadPool(threads, r -> { + final Thread t = new Thread(r, "postgres-flow-query"); + t.setDaemon(true); + return t; + }); + LOG.info("PostgresFlowQueryService started (dataSource={}, threads={}).", dataSourceName, threads); + } + + public void stop() { + if (executor != null) { + executor.shutdownNow(); + } + } + + private CompletableFuture async(final Supplier supplier) { + return CompletableFuture.supplyAsync(supplier, executor); + } + + // --------------------------------------------------------------------- + // Filter -> WHERE + // --------------------------------------------------------------------- + + /** Accumulated WHERE clause + bind params + the required time window. */ + private static final class Where { + final StringBuilder sql = new StringBuilder(); + final List params = new ArrayList<>(); + long start; + long end; + Integer snmpInterfaceId; + } + + private static Where where(final List filters) { + final Where w = new Where(); + boolean haveTime = false; + for (final Filter filter : filters) { + if (filter instanceof TimeRangeFilter) { + final TimeRangeFilter t = (TimeRangeFilter) filter; + w.start = t.getStart(); + w.end = t.getEnd(); + haveTime = true; + // Row selection on flow_ts (@timestamp equivalent) -> partition pruning. + w.sql.append(" AND f.flow_ts >= to_timestamp(").append(w.start).append("/1000.0)") + .append(" AND f.flow_ts <= to_timestamp(").append(w.end).append("/1000.0)"); + } else if (filter instanceof SnmpInterfaceIdFilter) { + final int id = ((SnmpInterfaceIdFilter) filter).getSnmpInterfaceId(); + w.snmpInterfaceId = id; + w.sql.append(" AND (f.input_snmp = ").append(id).append(" OR f.output_snmp = ").append(id).append(")"); + } else if (filter instanceof DscpFilter) { + final List dscp = ((DscpFilter) filter).getDscp(); + if (dscp != null && !dscp.isEmpty()) { + final StringBuilder in = new StringBuilder(); + for (int i = 0; i < dscp.size(); i++) { + if (i > 0) in.append(','); + in.append(dscp.get(i).intValue()); + } + w.sql.append(" AND f.dscp IN (").append(in).append(")"); + } + } else if (filter instanceof ExporterNodeFilter) { + final org.opennms.netmgt.flows.filter.api.NodeCriteria c = ((ExporterNodeFilter) filter).getCriteria(); + if (c.getNodeId() != null) { + w.sql.append(" AND f.exporter_node_id = ").append(c.getNodeId().intValue()); + } else { + w.sql.append(" AND f.document->'node_exporter'->>'foreign_source' = ?") + .append(" AND f.document->'node_exporter'->>'foreign_id' = ?"); + w.params.add(c.getForeignSource()); + w.params.add(c.getForeignId()); + } + } + } + if (!haveTime) { + throw new IllegalArgumentException("A TimeRangeFilter is required."); + } + return w; + } + + // --------------------------------------------------------------------- + // Flow count + // --------------------------------------------------------------------- + + @Override + public CompletableFuture getFlowCount(final List filters) { + return async(() -> { + final Where w = where(filters); + final String sql = "SELECT count(*) FROM flow f WHERE 1=1" + w.sql; + final Long n = jdbcTemplate.queryForObject(sql, Long.class, w.params.toArray()); + return n != null ? n : 0L; + }); + } + + // --------------------------------------------------------------------- + // Generic summary/series over a group expression + // --------------------------------------------------------------------- + + private static final class InOut { + long in; + long out; + boolean congestionEncountered; + boolean nonEcnCapableTransport; + } + + // ECN parity: a flow marks Congestion Encountered when its ecn codepoint is 3 (CE), and + // Not-ECT (non-ECN-capable transport) when its ecn codepoint is 0. Elastic's query builder + // OVERWRITES these per direction bucket (order-dependent); we take the union (bool_or) across + // the entity's flows, which is the intended "did any flow see CE / Not-ECT?" semantics. + private static final String ECN_CONGESTION = "bool_or((f.document->>'netflow.ecn') = '3')"; + private static final String ECN_NON_ECT = "bool_or((f.document->>'netflow.ecn') = '0')"; + + // Direction handling, EXACTLY as ElasticFlowRepository / RawFlowQueryService: + // * With NO SnmpInterfaceIdFilter, the Elastic query template constrains the aggregation to + // netflow.direction in {ingress,egress}, so UNKNOWN-direction flows are excluded from the whole + // summary/series (grouping, sums and ECN alike) -> directionConstraint() renders that filter. + // * With a SnmpInterfaceIdFilter, Elastic's unknownDirectionScript reclassifies an UNKNOWN flow to + // ingress when input_snmp matches the interface, else egress when output_snmp matches (an already + // explicit ingress/egress is kept), and nothing is dropped -> effectiveDirection() renders that CASE. + // RawFlowQueryService.isIngress() then maps the resulting direction ingress->in / egress->out and + // throws on anything else; here accumulate() simply skips anything that is not INGRESS/EGRESS. + + /** The direction a row is attributed to: the stored value, or (under an interface filter) UNKNOWN reclassified. */ + private static String effectiveDirection(final Where w) { + if (w.snmpInterfaceId == null) { + return "f.direction"; + } + final int id = w.snmpInterfaceId; + return "(CASE WHEN f.direction <> 'UNKNOWN' THEN f.direction " + + "WHEN f.input_snmp = " + id + " THEN 'INGRESS' " + + "WHEN f.output_snmp = " + id + " THEN 'EGRESS' END)"; + } + + /** Extra WHERE that drops UNKNOWN-direction flows when there is no interface filter to reclassify them. */ + private static String directionConstraint(final Where w) { + return w.snmpInterfaceId == null ? " AND f.direction IN ('INGRESS','EGRESS')" : ""; + } + + /** + * Returns entity -> (bytesIn,bytesOut) prorated over the window, ordered by total desc. + * @param entityExpr SQL expression producing the group key (aliased column) + * @param fromExtra extra FROM/JOIN (e.g. host unnest); may be empty + * @param topN if > 0, limit to the top N by total bytes; else all (subject to explicit) + * @param explicit if non-null, restrict to these entity keys + */ + private Map summaries(final String entityExpr, final String fromExtra, + final Where w, final Integer topN, final Set explicit) { + final long s = w.start; + final long e = w.end; + final String prorated = proratedForWindow(s, e); + final String effDir = effectiveDirection(w); + final List params = new ArrayList<>(w.params); + final StringBuilder sql = new StringBuilder(); + sql.append("SELECT ").append(entityExpr).append(" AS entity,") + .append(" COALESCE(SUM(").append(prorated).append(") FILTER (WHERE ").append(effDir).append(" = 'INGRESS'),0) AS bin,") + .append(" COALESCE(SUM(").append(prorated).append(") FILTER (WHERE ").append(effDir).append(" = 'EGRESS'),0) AS bout,") + .append(' ').append(ECN_CONGESTION).append(" AS cong,") + .append(' ').append(ECN_NON_ECT).append(" AS nonect") + .append(" FROM flow f").append(fromExtra) + .append(" WHERE 1=1").append(w.sql).append(directionConstraint(w)) + .append(" AND ").append(entityExpr).append(" IS NOT NULL"); + if (explicit != null) { + sql.append(" AND ").append(entityExpr).append(" IN ("); + int i = 0; + for (final String key : explicit) { + if (i++ > 0) sql.append(','); + sql.append('?'); + params.add(key); + } + sql.append(')'); + } + sql.append(" GROUP BY entity ORDER BY (COALESCE(SUM(").append(prorated).append("),0)) DESC"); + if (topN != null && topN > 0) { + sql.append(" LIMIT ").append(topN); + } + final Map out = new LinkedHashMap<>(); + jdbcTemplate.query(sql.toString(), rs -> { + final InOut io = new InOut(); + io.in = rs.getLong("bin"); + io.out = rs.getLong("bout"); + io.congestionEncountered = rs.getBoolean("cong"); + io.nonEcnCapableTransport = rs.getBoolean("nonect"); + out.put(rs.getString("entity"), io); + }, params.toArray()); + return out; + } + + /** Total prorated bytes over the whole window (for the "Other" bucket). */ + private InOut totals(final String fromExtra, final Where w, final String entityNotNull) { + final long s = w.start; + final long e = w.end; + final String prorated = proratedForWindow(s, e); + final String effDir = effectiveDirection(w); + final String sql = "SELECT " + + " COALESCE(SUM(" + prorated + ") FILTER (WHERE " + effDir + " = 'INGRESS'),0) AS bin," + + " COALESCE(SUM(" + prorated + ") FILTER (WHERE " + effDir + " = 'EGRESS'),0) AS bout" + + " FROM flow f" + fromExtra + " WHERE 1=1" + w.sql + directionConstraint(w) + + (entityNotNull != null ? " AND " + entityNotNull : ""); + final InOut io = new InOut(); + jdbcTemplate.query(sql, rs -> { + io.in = rs.getLong("bin"); + io.out = rs.getLong("bout"); + }, w.params.toArray()); + return io; + } + + private static void addOther(final Map ordered, final InOut total) { + long sumIn = 0, sumOut = 0; + for (final InOut io : ordered.values()) { + sumIn += io.in; + sumOut += io.out; + } + final InOut other = new InOut(); + other.in = Math.max(0, total.in - sumIn); + other.out = Math.max(0, total.out - sumOut); + ordered.put(OTHER, other); + } + + /** Histogram bucket alignment origin. Elastic's proportional_sum always uses offset 0 (epoch-aligned). */ + private static final long DEFAULT_ORIGIN = 0L; + + /** + * FROM ... CROSS JOIN LATERAL that expands each row into its overlapping buckets. Buckets align to + * {@code origin} + k*{@code step} (floor toward -inf, matching date_bin / the drift plugin's rounding); + * with {@code origin == 0} this is epoch-aligned, which is Elastic's only mode. + */ + private static String seriesLateral(final String fromExtra, final long step, final long origin, + final long s, final long e) { + return " FROM flow f" + fromExtra + + " CROSS JOIN LATERAL generate_series(" + + " (FLOOR((GREATEST(f.delta_switched, " + s + ") - " + origin + ")::numeric / " + step + ") * " + + step + " + " + origin + ")::bigint," + + " LEAST(f.last_switched, " + e + ")," + + " " + step + ") AS b(bucket_ms)"; + } + + private void accumulate(final Table table, final String entity, + final String direction, final long bucket, final double v) { + // Mirror RawFlowQueryService.isIngress: ingress -> in, egress -> out, anything else not + // attributable. UNKNOWN is excluded upstream (no interface filter) or reclassified to + // ingress/egress (interface filter present), so it never reaches here. + final int idx; + if ("INGRESS".equals(direction)) { + idx = 0; + } else if ("EGRESS".equals(direction)) { + idx = 1; + } else { + return; + } + double[] cell = table.get(entity, bucket); + if (cell == null) { + cell = new double[2]; + table.put(entity, bucket, cell); + } + cell[idx] += v; + } + + /** + * entity/direction/bucket -> summed prorated value, for a series. When {@code includeOther}, an + * extra {@link #OTHER} row aggregates every in-window row whose key is NOT among the selected keys + * (mirroring Elastic's must_not "others" query — always non-negative, no cross-bucket subtraction). + */ + private Table series(final String entityExpr, final String fromExtra, + final Where w, final long step, final long origin, + final Integer topN, final Set explicit, + final boolean includeOther) { + // Restrict the series to the same entities the summary would select (top N or explicit set). + final Set keys = (explicit != null) ? explicit : summaries(entityExpr, fromExtra, w, topN, null).keySet(); + final Table table = HashBasedTable.create(); + final long s = w.start; + final long e = w.end; + final String bucketExpr = proratedForBucket(step); + + if (!keys.isEmpty()) { + final List params = new ArrayList<>(w.params); + final StringBuilder in = new StringBuilder(); + int i = 0; + for (final String key : keys) { + if (i++ > 0) in.append(','); + in.append('?'); + params.add(key); + } + final String sql = + "SELECT entity, direction, bucket_ms, SUM(contribution) AS v FROM (" + + " SELECT " + entityExpr + " AS entity, " + effectiveDirection(w) + " AS direction, b.bucket_ms AS bucket_ms, " + + bucketExpr + " AS contribution" + + seriesLateral(fromExtra, step, origin, s, e) + + " WHERE 1=1" + w.sql + directionConstraint(w) + + " AND " + entityExpr + " IS NOT NULL" + + " AND " + entityExpr + " IN (" + in + ")" + + ") t GROUP BY entity, direction, bucket_ms"; + jdbcTemplate.query(sql, rs -> { accumulate(table, rs.getString("entity"), + rs.getString("direction"), rs.getLong("bucket_ms"), rs.getDouble("v")); }, + params.toArray()); + } + + if (includeOther) { + final List params = new ArrayList<>(w.params); + final StringBuilder notIn = new StringBuilder(); + for (final String key : keys) { + notIn.append(notIn.length() == 0 ? "" : ",").append('?'); + params.add(key); + } + final String excludeSelected = keys.isEmpty() ? "" : " AND " + entityExpr + " NOT IN (" + notIn + ")"; + final String sql = + "SELECT direction, bucket_ms, SUM(contribution) AS v FROM (" + + " SELECT " + effectiveDirection(w) + " AS direction, b.bucket_ms AS bucket_ms, " + bucketExpr + " AS contribution" + + seriesLateral(fromExtra, step, origin, s, e) + + " WHERE 1=1" + w.sql + directionConstraint(w) + + " AND " + entityExpr + " IS NOT NULL" + excludeSelected + + ") t GROUP BY direction, bucket_ms"; + jdbcTemplate.query(sql, rs -> { accumulate(table, OTHER, + rs.getString("direction"), rs.getLong("bucket_ms"), rs.getDouble("v")); }, + params.toArray()); + } + return table; + } + + // --------------------------------------------------------------------- + // Applications + // --------------------------------------------------------------------- + + @Override + public CompletableFuture> getApplications(final String matchingPrefix, final long limit, final List filters) { + return async(() -> { + final Where w = where(filters); + final StringBuilder sql = new StringBuilder("SELECT DISTINCT f.application FROM flow f WHERE 1=1") + .append(w.sql).append(" AND f.application IS NOT NULL"); + final List params = new ArrayList<>(w.params); + if (matchingPrefix != null && !matchingPrefix.isEmpty()) { + sql.append(" AND f.application ILIKE ?"); + params.add(matchingPrefix + "%"); + } + sql.append(" ORDER BY f.application"); + if (limit > 0) sql.append(" LIMIT ").append(limit); + return jdbcTemplate.queryForList(sql.toString(), String.class, params.toArray()); + }); + } + + @Override + public CompletableFuture>> getTopNApplicationSummaries(final int n, final boolean includeOther, final List filters) { + return async(() -> stringSummaries("f.application", "", where(filters), n, null, includeOther, "f.application IS NOT NULL")); + } + + @Override + public CompletableFuture>> getApplicationSummaries(final Set applications, final boolean includeOther, final List filters) { + return async(() -> stringSummaries("f.application", "", where(filters), null, applications, includeOther, "f.application IS NOT NULL")); + } + + @Override + public CompletableFuture, Long, Double>> getApplicationSeries(final Set applications, final long step, final boolean includeOther, final List filters) { + return async(() -> stringSeries("f.application", "", where(filters), step, null, applications, includeOther)); + } + + @Override + public CompletableFuture, Long, Double>> getTopNApplicationSeries(final int n, final long step, final boolean includeOther, final List filters) { + return async(() -> stringSeries("f.application", "", where(filters), step, n, null, includeOther)); + } + + // --------------------------------------------------------------------- + // Conversations + // --------------------------------------------------------------------- + + @Override + public CompletableFuture> getConversations(final String locationPattern, final String protocolPattern, + final String lowerIPPattern, final String upperIPPattern, + final String applicationPattern, final long limit, final List filters) { + return async(() -> { + final Where w = where(filters); + final String sql = "SELECT DISTINCT f.convo_key FROM flow f WHERE 1=1" + w.sql + " AND f.convo_key IS NOT NULL"; + final List keys = jdbcTemplate.queryForList(sql, String.class, w.params.toArray()); + final List result = new ArrayList<>(); + for (final String key : keys) { + final ConversationKey ck = ConversationKeyUtils.fromJsonString(key); + if (ck == null) continue; + if (!matches(locationPattern, ck.getLocation())) continue; + if (!matches(protocolPattern, String.valueOf(ck.getProtocol()))) continue; + if (!matches(lowerIPPattern, ck.getLowerIp())) continue; + if (!matches(upperIPPattern, ck.getUpperIp())) continue; + if (!matches(applicationPattern, ck.getApplication())) continue; + result.add(key); + if (limit > 0 && result.size() >= limit) break; + } + return result; + }); + } + + private static boolean matches(final String pattern, final String value) { + if (pattern == null || pattern.isEmpty() || ".*".equals(pattern)) { + return true; + } + return value != null && value.matches(pattern); + } + + @Override + public CompletableFuture>> getTopNConversationSummaries(final int n, final boolean includeOther, final List filters) { + return async(() -> conversationSummaries(where(filters), n, null, includeOther)); + } + + @Override + public CompletableFuture>> getConversationSummaries(final Set conversations, final boolean includeOther, final List filters) { + return async(() -> conversationSummaries(where(filters), null, conversations, includeOther)); + } + + @Override + public CompletableFuture, Long, Double>> getConversationSeries(final Set conversations, final long step, final boolean includeOther, final List filters) { + return async(() -> mapSeries(series("f.convo_key", "", where(filters), step, DEFAULT_ORIGIN, null, conversations, includeOther), PostgresFlowQueryService::toConversation)); + } + + @Override + public CompletableFuture, Long, Double>> getTopNConversationSeries(final int n, final long step, final boolean includeOther, final List filters) { + return async(() -> mapSeries(series("f.convo_key", "", where(filters), step, DEFAULT_ORIGIN, n, null, includeOther), PostgresFlowQueryService::toConversation)); + } + + private List> conversationSummaries(final Where w, final Integer topN, final Set explicit, final boolean includeOther) { + final Map m = summaries("f.convo_key", "", w, topN, explicit); + if (includeOther) addOther(m, totals("", w, "f.convo_key IS NOT NULL")); + final List> out = new ArrayList<>(); + for (final Map.Entry en : m.entrySet()) { + out.add(summaryBuilder(toConversation(en.getKey()), en.getValue()).build()); + } + return out; + } + + private static Conversation toConversation(final String key) { + if (OTHER.equals(key)) { + return Conversation.forOther().build(); + } + final ConversationKey ck = ConversationKeyUtils.fromJsonString(key); + return Conversation.from(ck).build(); + } + + // --------------------------------------------------------------------- + // Hosts (a flow contributes to BOTH its src and dst host) + // --------------------------------------------------------------------- + + private static final String HOST_FROM = " CROSS JOIN LATERAL (VALUES (host(f.src_addr)), (host(f.dst_addr))) AS h(host)"; + private static final String HOST_EXPR = "h.host"; + + @Override + public CompletableFuture> getHosts(final String regex, final long limit, final List filters) { + return async(() -> { + final Where w = where(filters); + final StringBuilder sql = new StringBuilder("SELECT DISTINCT " + HOST_EXPR + " FROM flow f" + HOST_FROM + " WHERE 1=1") + .append(w.sql).append(" AND ").append(HOST_EXPR).append(" IS NOT NULL"); + final List params = new ArrayList<>(w.params); + if (regex != null && !regex.isEmpty() && !".*".equals(regex)) { + sql.append(" AND ").append(HOST_EXPR).append(" ~ ?"); + params.add(regex); + } + sql.append(" ORDER BY ").append(HOST_EXPR); + if (limit > 0) sql.append(" LIMIT ").append(limit); + return jdbcTemplate.queryForList(sql.toString(), String.class, params.toArray()); + }); + } + + @Override + public CompletableFuture>> getTopNHostSummaries(final int n, final boolean includeOther, final List filters) { + return async(() -> hostSummaries(where(filters), n, null, includeOther)); + } + + @Override + public CompletableFuture>> getHostSummaries(final Set hosts, final boolean includeOther, final List filters) { + return async(() -> hostSummaries(where(filters), null, hosts, includeOther)); + } + + @Override + public CompletableFuture, Long, Double>> getHostSeries(final Set hosts, final long step, final boolean includeOther, final List filters) { + return async(() -> mapSeries(series(HOST_EXPR, HOST_FROM, where(filters), step, DEFAULT_ORIGIN, null, hosts, includeOther), PostgresFlowQueryService::toHost)); + } + + @Override + public CompletableFuture, Long, Double>> getTopNHostSeries(final int n, final long step, final boolean includeOther, final List filters) { + return async(() -> mapSeries(series(HOST_EXPR, HOST_FROM, where(filters), step, DEFAULT_ORIGIN, n, null, includeOther), PostgresFlowQueryService::toHost)); + } + + private List> hostSummaries(final Where w, final Integer topN, final Set explicit, final boolean includeOther) { + final Map m = summaries(HOST_EXPR, HOST_FROM, w, topN, explicit); + if (includeOther) addOther(m, totals(HOST_FROM, w, HOST_EXPR + " IS NOT NULL")); + final List> out = new ArrayList<>(); + for (final Map.Entry en : m.entrySet()) { + out.add(summaryBuilder(toHost(en.getKey()), en.getValue()).build()); + } + return out; + } + + private static Host toHost(final String ip) { + return OTHER.equals(ip) ? Host.forOther().build() : Host.from(ip).build(); + } + + // --------------------------------------------------------------------- + // Generic field queries (LimitedCardinalityField) + // --------------------------------------------------------------------- + + private static String fieldExpr(final LimitedCardinalityField field) { + // DSCP is promoted to a real column; any other limited-cardinality field falls back to its + // jsonb value under the same netflow.* key the Elastic schema uses (field.fieldName). + if (field == LimitedCardinalityField.DSCP) { + return "f.dscp::text"; + } + return "(f.document->>'" + field.fieldName + "')"; + } + + @Override + public CompletableFuture> getFieldValues(final LimitedCardinalityField field, final List filters) { + return async(() -> { + final Where w = where(filters); + final String expr = fieldExpr(field); + final String sql = "SELECT DISTINCT " + expr + " FROM flow f WHERE 1=1" + w.sql + " AND " + expr + " IS NOT NULL ORDER BY 1"; + return jdbcTemplate.queryForList(sql, String.class, w.params.toArray()); + }); + } + + @Override + public CompletableFuture>> getFieldSummaries(final LimitedCardinalityField field, final List filters) { + return async(() -> stringSummaries(fieldExpr(field), "", where(filters), null, null, false, fieldExpr(field) + " IS NOT NULL")); + } + + @Override + public CompletableFuture, Long, Double>> getFieldSeries(final LimitedCardinalityField field, final long step, final List filters) { + return async(() -> { + // All distinct field values become the "explicit" set (limited cardinality by definition). + final Where w = where(filters); + final String expr = fieldExpr(field); + final List values = jdbcTemplate.queryForList( + "SELECT DISTINCT " + expr + " FROM flow f WHERE 1=1" + w.sql + " AND " + expr + " IS NOT NULL", + String.class, w.params.toArray()); + return mapSeries(series(expr, "", w, step, DEFAULT_ORIGIN, null, new java.util.HashSet<>(values), false), s -> s); + }); + } + + // --------------------------------------------------------------------- + // String-entity summary/series assembly + Table mapping + // --------------------------------------------------------------------- + + private List> stringSummaries(final String entityExpr, final String fromExtra, final Where w, + final Integer topN, final Set explicit, + final boolean includeOther, final String entityNotNull) { + final Map m = summaries(entityExpr, fromExtra, w, topN, explicit); + if (includeOther) addOther(m, totals(fromExtra, w, entityNotNull)); + final List> out = new ArrayList<>(); + for (final Map.Entry en : m.entrySet()) { + out.add(summaryBuilder(en.getKey(), en.getValue()).build()); + } + return out; + } + + /** Assemble a TrafficSummary from an entity + its prorated bytes and ECN flags. */ + private static TrafficSummary.Builder summaryBuilder(final T entity, final InOut io) { + return TrafficSummary.from(entity) + .withBytes(io.in, io.out) + .withCongestionEncountered(io.congestionEncountered) + .withNonEcnCapableTransport(io.nonEcnCapableTransport); + } + + private Table, Long, Double> stringSeries(final String entityExpr, final String fromExtra, final Where w, + final long step, final Integer topN, final Set explicit, + final boolean includeOther) { + return mapSeries(series(entityExpr, fromExtra, w, step, DEFAULT_ORIGIN, topN, explicit, includeOther), s -> s); + } + + private Table, Long, Double> mapSeries(final Table raw, + final java.util.function.Function toEntity) { + final Table, Long, Double> out = HashBasedTable.create(); + for (final Table.Cell cell : raw.cellSet()) { + final T entity = toEntity.apply(cell.getRowKey()); + final double[] v = cell.getValue(); + if (v[0] != 0.0) out.put(new Directional<>(entity, true), cell.getColumnKey(), v[0]); + if (v[1] != 0.0) out.put(new Directional<>(entity, false), cell.getColumnKey(), v[1]); + } + return out; + } + + // --- config setters (blueprint) --- + public void setDataSourceName(final String dataSourceName) { this.dataSourceName = Objects.requireNonNull(dataSourceName); } + public void setThreads(final int threads) { this.threads = threads; } + /** Test/embedding hook: use this DataSource directly instead of resolving one from DataSourceFactory. */ + public void setDataSource(final DataSource dataSource) { this.dataSource = dataSource; } +} \ No newline at end of file diff --git a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepository.java b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepository.java new file mode 100644 index 000000000000..082ed953510f --- /dev/null +++ b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepository.java @@ -0,0 +1,181 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.postgres; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Types; +import java.util.Collection; +import java.util.List; +import java.util.Objects; + +import javax.sql.DataSource; + +import org.opennms.core.db.DataSourceFactory; +import org.opennms.integration.api.v1.flows.Flow; +import org.opennms.integration.api.v1.flows.FlowException; +import org.opennms.integration.api.v1.flows.FlowRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.jdbc.core.BatchPreparedStatementSetter; +import org.springframework.jdbc.core.JdbcTemplate; + +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.databind.ObjectMapper; + +import liquibase.Liquibase; +import liquibase.database.DatabaseFactory; +import liquibase.database.jvm.JdbcConnection; +import liquibase.resource.ClassLoaderResourceAccessor; + +/** + * Prototype {@link FlowRepository} that persists enriched flows to PostgreSQL (hot columns + jsonb). + * + *

Targets the internal "opennms" datasource by default; set {@code dataSourceName} to a named + * datasource (defined in {@code opennms-datasources.xml}) to persist to an external PostgreSQL for + * scale. Writes are batched with a drop-newest backpressure policy (see {@link BatchingFlowWriter}). + */ +public class PostgresFlowRepository implements FlowRepository { + + private static final Logger LOG = LoggerFactory.getLogger(PostgresFlowRepository.class); + + private static final String CHANGELOG = "org/opennms/netmgt/flows/postgres/changelog.xml"; + + private static final String INSERT_SQL = + "INSERT INTO flow (flow_ts, delta_switched, last_switched, first_switched, bytes, packets, " + + "sampling_interval, direction, application, convo_key, src_addr, dst_addr, protocol, dscp, " + + "exporter_node_id, input_snmp, output_snmp, location, document) VALUES " + + "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::inet, ?::inet, ?, ?, ?, ?, ?, ?, ?::jsonb)"; + + private final MetricRegistry metrics; + private final ObjectMapper objectMapper = new ObjectMapper(); + private final FlowRowMapper rowMapper = new FlowRowMapper(objectMapper); + + private String dataSourceName = "opennms"; + private int batchSize = 1000; + private long flushIntervalMs = 500; + private int queueCapacity = 100_000; + private boolean runSchemaChangelog = true; + + private DataSource dataSource; + private JdbcTemplate jdbcTemplate; + private BatchingFlowWriter writer; + + public PostgresFlowRepository(final MetricRegistry metrics) { + this.metrics = Objects.requireNonNull(metrics); + } + + public void start() throws Exception { + if (this.dataSource == null) { + DataSourceFactory.init(dataSourceName); + this.dataSource = DataSourceFactory.getInstance(dataSourceName); + } + this.jdbcTemplate = new JdbcTemplate(dataSource); + if (runSchemaChangelog) { + installSchema(); + } + this.writer = new BatchingFlowWriter<>("postgresFlowRepository", queueCapacity, batchSize, + flushIntervalMs, this::flush, metrics); + this.writer.start(); + LOG.info("PostgresFlowRepository started (dataSource={}, batchSize={}, flushIntervalMs={}, queueCapacity={}).", + dataSourceName, batchSize, flushIntervalMs, queueCapacity); + } + + public void stop() { + if (writer != null) { + writer.close(); + } + LOG.info("PostgresFlowRepository stopped."); + } + + @Override + public void persist(final Collection flows) throws FlowException { + if (flows == null || flows.isEmpty()) { + return; + } + for (final Flow flow : flows) { + writer.add(rowMapper.toRow(flow)); + } + } + + private void flush(final List rows) { + jdbcTemplate.batchUpdate(INSERT_SQL, new BatchPreparedStatementSetter() { + @Override + public void setValues(final PreparedStatement ps, final int i) throws SQLException { + final FlowRow r = rows.get(i); + int c = 1; + ps.setTimestamp(c++, r.flowTs); + setLong(ps, c++, r.deltaSwitched); + setLong(ps, c++, r.lastSwitched); + setLong(ps, c++, r.firstSwitched); + setLong(ps, c++, r.bytes); + setLong(ps, c++, r.packets); + if (r.samplingInterval != null) ps.setDouble(c++, r.samplingInterval); else ps.setNull(c++, Types.DOUBLE); + ps.setString(c++, r.direction); + ps.setString(c++, r.application); + ps.setString(c++, r.convoKey); + ps.setString(c++, r.srcAddr); + ps.setString(c++, r.dstAddr); + setInt(ps, c++, r.protocol); + setInt(ps, c++, r.dscp); + setInt(ps, c++, r.exporterNodeId); + setInt(ps, c++, r.inputSnmp); + setInt(ps, c++, r.outputSnmp); + ps.setString(c++, r.location); + ps.setString(c, r.documentJson); + } + + @Override + public int getBatchSize() { + return rows.size(); + } + }); + } + + private void installSchema() throws Exception { + try (java.sql.Connection conn = dataSource.getConnection()) { + final liquibase.database.Database db = DatabaseFactory.getInstance() + .findCorrectDatabaseImplementation(new JdbcConnection(conn)); + final Liquibase liquibase = new Liquibase(CHANGELOG, + new ClassLoaderResourceAccessor(getClass().getClassLoader()), db); + liquibase.update(""); + } + LOG.info("PostgresFlowRepository schema ensured on datasource {}.", dataSourceName); + } + + private static void setLong(final PreparedStatement ps, final int idx, final Long v) throws SQLException { + if (v != null) ps.setLong(idx, v); else ps.setNull(idx, Types.BIGINT); + } + + private static void setInt(final PreparedStatement ps, final int idx, final Integer v) throws SQLException { + if (v != null) ps.setInt(idx, v); else ps.setNull(idx, Types.INTEGER); + } + + // --- config setters (blueprint) --- + /** Test/embedding hook: use this DataSource directly instead of resolving one from DataSourceFactory. */ + public void setDataSource(final DataSource dataSource) { this.dataSource = dataSource; } + public void setDataSourceName(final String dataSourceName) { this.dataSourceName = dataSourceName; } + public void setBatchSize(final int batchSize) { this.batchSize = batchSize; } + public void setFlushIntervalMs(final long flushIntervalMs) { this.flushIntervalMs = flushIntervalMs; } + public void setQueueCapacity(final int queueCapacity) { this.queueCapacity = queueCapacity; } + public void setRunSchemaChangelog(final boolean runSchemaChangelog) { this.runSchemaChangelog = runSchemaChangelog; } +} \ No newline at end of file diff --git a/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml new file mode 100644 index 000000000000..68b7fb9a6bfc --- /dev/null +++ b/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/features/flows/postgres/src/main/resources/org/opennms/netmgt/flows/postgres/changelog.xml b/features/flows/postgres/src/main/resources/org/opennms/netmgt/flows/postgres/changelog.xml new file mode 100644 index 000000000000..21ad9b9db7dd --- /dev/null +++ b/features/flows/postgres/src/main/resources/org/opennms/netmgt/flows/postgres/changelog.xml @@ -0,0 +1,120 @@ + + + + + + + + + + ) + application text, -- netflow.application + convo_key text, -- netflow.convo_key + src_addr inet, + dst_addr inet, + protocol smallint, + dscp smallint, -- DscpFilter + LimitedCardinalityField.DSCP + exporter_node_id integer, -- ExporterNodeFilter + input_snmp integer, -- SnmpInterfaceIdFilter (input) + output_snmp integer, -- SnmpInterfaceIdFilter (output) + location text, + document jsonb NOT NULL -- full enriched flow + ) PARTITION BY RANGE (flow_ts); + ]]> + DROP TABLE IF EXISTS flow; + + + + + + + + + retention_days))::date; + child record; + child_date date; + BEGIN + -- Pre-create today + upcoming daily partitions. + FOR i IN 0..premake_days LOOP + d := (now()::date + i); + part_name := format('flow_p%s', to_char(d, 'YYYYMMDD')); + IF NOT EXISTS (SELECT 1 FROM pg_class WHERE relname = part_name) THEN + BEGIN + EXECUTE format( + 'CREATE TABLE %I PARTITION OF flow FOR VALUES FROM (%L) TO (%L)', + part_name, d::timestamptz, (d + 1)::timestamptz); + EXCEPTION WHEN others THEN + RAISE NOTICE 'flow_maintain_partitions: could not create %: %', part_name, SQLERRM; + END; + END IF; + END LOOP; + + -- Drop partitions whose day is older than the retention cutoff. + FOR child IN + SELECT c.relname AS relname + FROM pg_inherits i + JOIN pg_class c ON c.oid = i.inhrelid + JOIN pg_class p ON p.oid = i.inhparent + WHERE p.relname = 'flow' AND c.relname ~ '^flow_p[0-9]{8}$' + LOOP + child_date := to_date(right(child.relname, 8), 'YYYYMMDD'); + IF child_date < cutoff THEN + EXECUTE format('DROP TABLE IF EXISTS %I', child.relname); + END IF; + END LOOP; + END; + $$ LANGUAGE plpgsql; + ]]> + DROP FUNCTION IF EXISTS flow_maintain_partitions(int, int); + + + + + + + + diff --git a/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepositoryIT.java b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepositoryIT.java new file mode 100644 index 000000000000..87e0a73dd46c --- /dev/null +++ b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepositoryIT.java @@ -0,0 +1,295 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.postgres; + +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import javax.sql.DataSource; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.Mockito; +import org.opennms.integration.api.v1.flows.Flow; +import org.opennms.netmgt.flows.api.Conversation; +import org.opennms.netmgt.flows.api.Directional; +import org.opennms.netmgt.flows.api.Host; +import org.opennms.netmgt.flows.api.TrafficSummary; +import org.opennms.netmgt.flows.filter.api.Filter; +import org.opennms.netmgt.flows.filter.api.SnmpInterfaceIdFilter; +import org.opennms.netmgt.flows.filter.api.TimeRangeFilter; +import org.postgresql.ds.PGSimpleDataSource; +import org.springframework.jdbc.core.JdbcTemplate; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.PostgreSQLContainer; + +import com.codahale.metrics.MetricRegistry; +import com.google.common.collect.Table; + +/** + * End-to-end verification of {@link PostgresFlowRepository} and {@link PostgresFlowQueryService} + * against a real PostgreSQL. By default it starts a {@code postgres:15} Testcontainer; pass + * {@code -Dit.postgres.jdbcUrl=...} (with optional {@code it.postgres.user}/{@code it.postgres.password}) + * to run against an already-running instance and skip Testcontainers entirely. + */ +public class PostgresFlowRepositoryIT { + private static final long BASE = 1700000000000L; + private static final long WINDOW = 1000L; + private static PostgreSQLContainer POSTGRES; + private static PGSimpleDataSource DATA_SOURCE; + private static PostgresFlowRepository repository; + private static PostgresFlowQueryService queryService; + private static JdbcTemplate jdbc; + + @BeforeClass + public static void startContainer() throws Exception { + String password; + String user; + String url; + String externalUrl = System.getProperty("it.postgres.jdbcUrl"); + if (externalUrl != null && !externalUrl.isEmpty()) { + url = externalUrl; + user = System.getProperty("it.postgres.user", "postgres"); + password = System.getProperty("it.postgres.password", "postgres"); + } else { + Assume.assumeTrue("No Docker/podman runtime available; skipping live-SQL IT.", DockerClientFactory.instance().isDockerAvailable()); + POSTGRES = new PostgreSQLContainer<>("postgres:15"); + POSTGRES.start(); + url = POSTGRES.getJdbcUrl(); + user = POSTGRES.getUsername(); + password = POSTGRES.getPassword(); + } + DATA_SOURCE = new PGSimpleDataSource(); + DATA_SOURCE.setUrl(url); + DATA_SOURCE.setUser(user); + DATA_SOURCE.setPassword(password); + jdbc = new JdbcTemplate(DATA_SOURCE); + repository = new PostgresFlowRepository(new MetricRegistry()); + repository.setDataSource(DATA_SOURCE); + repository.setRunSchemaChangelog(true); + repository.setBatchSize(10); + repository.setFlushIntervalMs(50L); + repository.start(); + queryService = new PostgresFlowQueryService(); + queryService.setDataSource(DATA_SOURCE); + queryService.start(); + } + + @AfterClass + public static void stopContainer() { + if (queryService != null) { + queryService.stop(); + } + if (repository != null) { + repository.stop(); + } + if (POSTGRES != null) { + POSTGRES.stop(); + } + } + + @After + public void truncate() { + if (jdbc != null) { + jdbc.execute("TRUNCATE TABLE flow"); + } + } + + private static List window() { + return Collections.singletonList(new TimeRangeFilter(BASE, BASE + WINDOW)); + } + + private void insert(String application, String direction, long delta, long last, long bytes, Double sampling, String src, String dst, String convoKey) { + jdbc.update("INSERT INTO flow (flow_ts, delta_switched, last_switched, bytes, sampling_interval, direction, application, src_addr, dst_addr, convo_key, document) VALUES (to_timestamp(?/1000.0), ?, ?, ?, ?, ?, ?, ?::inet, ?::inet, ?, '{}'::jsonb)", + last, delta, last, bytes, sampling, direction, application, src, dst, convoKey); + } + + private void insertIf(String application, String direction, long delta, long last, long bytes, Integer inputSnmp, Integer outputSnmp) { + jdbc.update("INSERT INTO flow (flow_ts, delta_switched, last_switched, bytes, sampling_interval, direction, application, input_snmp, output_snmp, document) VALUES (to_timestamp(?/1000.0), ?, ?, ?, 1.0, ?, ?, ?, ?, '{}'::jsonb)", + last, delta, last, bytes, direction, application, inputSnmp, outputSnmp); + } + + private void insertEcn(String application, String direction, long delta, long last, long bytes, Integer ecn) { + Object document = ecn == null ? "{}" : "{\"netflow.ecn\": " + ecn + "}"; + jdbc.update("INSERT INTO flow (flow_ts, delta_switched, last_switched, bytes, sampling_interval, direction, application, document) VALUES (to_timestamp(?/1000.0), ?, ?, ?, 1.0, ?, ?, ?::jsonb)", + last, delta, last, bytes, direction, application, document); + } + + private static Map byEntity(List> summaries) { + HashMap m = new HashMap<>(); + for (TrafficSummary s : summaries) { + m.put(s.getEntity(), new long[]{s.getBytesIn(), s.getBytesOut()}); + } + return m; + } + + @Test + public void applicationSummariesMatchProration() throws Exception { + this.insert("http", "INGRESS", BASE, BASE + 1000L, 1000L, 1.0, "10.0.0.1", "10.0.0.2", null); + this.insert("http", "EGRESS", BASE + 250L, BASE + 750L, 400L, 1.0, "10.0.0.1", "10.0.0.2", null); + this.insert("http", "INGRESS", BASE + 500L, BASE + 500L, 300L, 1.0, "10.0.0.3", "10.0.0.4", null); + this.insert("http", "INGRESS", BASE + 100L, BASE + 900L, 2L, 4.0, "10.0.0.5", "10.0.0.6", null); + this.insert("https", "INGRESS", BASE - 1000L, BASE + 1000L, 2000L, 1.0, "10.0.0.7", "10.0.0.8", null); + Map got = byEntity(queryService.getTopNApplicationSummaries(10, false, window()).get(10L, TimeUnit.SECONDS)); + Assert.assertEquals(2L, got.size()); + assertArrayEquals2(new long[]{1308L, 400L}, got.get("http")); + assertArrayEquals2(new long[]{1000L, 0L}, got.get("https")); + } + + @Test + public void unknownDirectionExcludedWithoutInterfaceFilter() throws Exception { + // Exact Elastic parity: with no interface filter the aggregation is constrained to + // ingress/egress, so UNKNOWN-direction bytes are dropped (not counted as ingress) and an + // application whose flows are all UNKNOWN does not appear at all. + this.insert("dns", "UNKNOWN", BASE, BASE + 1000L, 500L, 1.0, "10.0.0.1", "10.0.0.2", null); + this.insert("dns", "EGRESS", BASE, BASE + 1000L, 300L, 1.0, "10.0.0.1", "10.0.0.2", null); + this.insert("onlyunknown", "UNKNOWN", BASE, BASE + 1000L, 400L, 1.0, "10.0.0.3", "10.0.0.4", null); + Map got = byEntity(queryService.getTopNApplicationSummaries(10, false, window()).get(10L, TimeUnit.SECONDS)); + assertArrayEquals2(new long[]{0L, 300L}, got.get("dns")); + Assert.assertFalse("an all-UNKNOWN application must be excluded without an interface filter", got.containsKey("onlyunknown")); + } + + @Test + public void unknownDirectionReclassifiedUnderInterfaceFilter() throws Exception { + // Exact Elastic parity: with a SnmpInterfaceIdFilter, an UNKNOWN flow is reclassified to + // ingress when input_snmp matches the interface, else egress when output_snmp matches. + this.insertIf("dns", "UNKNOWN", BASE, BASE + 1000L, 500L, 7, 99); // input_snmp=7 -> INGRESS + this.insertIf("dns", "UNKNOWN", BASE, BASE + 1000L, 200L, 99, 7); // output_snmp=7 -> EGRESS + List filters = Arrays.asList(new TimeRangeFilter(BASE, BASE + WINDOW), new SnmpInterfaceIdFilter(7)); + Map got = byEntity(queryService.getTopNApplicationSummaries(10, false, filters).get(10L, TimeUnit.SECONDS)); + assertArrayEquals2(new long[]{500L, 200L}, got.get("dns")); + } + + @Test + public void applicationSeriesMatchesProration() throws Exception { + this.insert("dns", "INGRESS", BASE, BASE + 1000L, 1000L, 1.0, "10.0.0.1", "10.0.0.2", null); + Table, Long, Double> series = queryService.getApplicationSeries(Collections.singleton("dns"), 500L, false, window()).get(10L, TimeUnit.SECONDS); + Directional ingress = new Directional<>("dns", true); + Assert.assertEquals(500.0, series.get(ingress, BASE), 1.0E-6); + Assert.assertEquals(500.0, series.get(ingress, BASE + 500L), 1.0E-6); + } + + @Test + public void hostSummariesCountBothEndpoints() throws Exception { + this.insert("x", "INGRESS", BASE, BASE + 1000L, 600L, 1.0, "192.168.1.1", "192.168.1.2", null); + HashMap got = new HashMap<>(); + for (TrafficSummary s : queryService.getTopNHostSummaries(10, false, window()).get(10L, TimeUnit.SECONDS)) { + got.put(s.getEntity().getIp(), new long[]{s.getBytesIn(), s.getBytesOut()}); + } + assertArrayEquals2(new long[]{600L, 0L}, got.get("192.168.1.1")); + assertArrayEquals2(new long[]{600L, 0L}, got.get("192.168.1.2")); + } + + @Test + public void conversationSummaryParsesConvoKey() throws Exception { + String convoKey = "[\"Default\",6,\"10.0.0.1\",\"10.0.0.2\",\"http\"]"; + this.insert("http", "INGRESS", BASE, BASE + 1000L, 800L, 1.0, "10.0.0.1", "10.0.0.2", convoKey); + List> convos = queryService.getConversationSummaries(Collections.singleton(convoKey), false, window()).get(10L, TimeUnit.SECONDS); + Assert.assertEquals(1L, convos.size()); + Assert.assertEquals(800L, convos.get(0).getBytesIn()); + Assert.assertEquals("http", convos.get(0).getEntity().getApplication()); + } + + @Test + public void topNApplicationSeriesIncludeOtherAggregatesNonTopN() throws Exception { + this.insert("top", "INGRESS", BASE, BASE + 1000L, 1000L, 1.0, "10.0.0.1", "10.0.0.2", null); + this.insert("a", "INGRESS", BASE, BASE + 1000L, 400L, 1.0, "10.0.0.1", "10.0.0.2", null); + this.insert("b", "INGRESS", BASE, BASE + 1000L, 600L, 1.0, "10.0.0.1", "10.0.0.2", null); + Table, Long, Double> series = queryService.getTopNApplicationSeries(1, 500L, true, window()).get(10L, TimeUnit.SECONDS); + Directional top = new Directional<>("top", true); + Directional other = new Directional<>("Other", true); + Assert.assertEquals(500.0, series.get(top, BASE), 1.0E-6); + Assert.assertEquals(500.0, series.get(top, BASE + 500L), 1.0E-6); + Assert.assertEquals(500.0, series.get(other, BASE), 1.0E-6); + Assert.assertEquals(500.0, series.get(other, BASE + 500L), 1.0E-6); + } + + @Test + public void applicationSummariesReportEcnFlags() throws Exception { + this.insertEcn("ce", "INGRESS", BASE, BASE + 1000L, 100L, 3); + this.insertEcn("noect", "INGRESS", BASE, BASE + 1000L, 100L, 0); + this.insertEcn("mixed", "INGRESS", BASE, BASE + 1000L, 100L, 3); + this.insertEcn("mixed", "INGRESS", BASE, BASE + 1000L, 100L, 0); + this.insertEcn("plain", "INGRESS", BASE, BASE + 1000L, 100L, 1); + HashMap ecn = new HashMap<>(); + for (TrafficSummary s : queryService.getTopNApplicationSummaries(10, false, window()).get(10L, TimeUnit.SECONDS)) { + ecn.put(s.getEntity(), new boolean[]{s.isCongestionEncountered(), s.isNonEcnCapableTransport()}); + } + Assert.assertArrayEquals(new boolean[]{true, false}, ecn.get("ce")); + Assert.assertArrayEquals(new boolean[]{false, true}, ecn.get("noect")); + Assert.assertArrayEquals(new boolean[]{true, true}, ecn.get("mixed")); + Assert.assertArrayEquals(new boolean[]{false, false}, ecn.get("plain")); + } + + @Test + public void flowCountHonorsTimeWindow() throws Exception { + this.insert("http", "INGRESS", BASE, BASE + 500L, 100L, 1.0, "10.0.0.1", "10.0.0.2", null); + this.insert("http", "INGRESS", BASE + 5000L, BASE + 6000L, 100L, 1.0, "10.0.0.1", "10.0.0.2", null); + Assert.assertEquals(Long.valueOf(1L), queryService.getFlowCount(window()).get(10L, TimeUnit.SECONDS)); + } + + @Test + public void persistWritesRowsThroughTheBatchingWriter() throws Exception { + repository.persist(Arrays.asList( + mockFlow("http", Flow.Direction.INGRESS, BASE + 100L, BASE + 900L, 1234L, "10.1.1.1", "10.1.1.2"), + mockFlow("http", Flow.Direction.EGRESS, BASE + 100L, BASE + 900L, 5678L, "10.1.1.2", "10.1.1.1"))); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10L); + Integer count = 0; + while (System.nanoTime() < deadline && ((count = jdbc.queryForObject("SELECT count(*) FROM flow", Integer.class)) == null || count != 2)) { + Thread.sleep(50L); + } + Assert.assertEquals(Integer.valueOf(2), count); + String app = jdbc.queryForObject("SELECT document->>'netflow.application' FROM flow WHERE direction = 'INGRESS'", String.class); + Assert.assertEquals("http", app); + long total = jdbc.queryForObject("SELECT COALESCE(SUM(bytes),0) FROM flow", Long.class); + Assert.assertEquals(6912L, total); + } + + private static Flow mockFlow(String application, Flow.Direction direction, long delta, long last, long bytes, String src, String dst) { + Flow flow = Mockito.mock(Flow.class); + Mockito.lenient().when(flow.getTimestamp()).thenReturn(Instant.ofEpochMilli(last)); + Mockito.lenient().when(flow.getDeltaSwitched()).thenReturn(Instant.ofEpochMilli(delta)); + Mockito.lenient().when(flow.getLastSwitched()).thenReturn(Instant.ofEpochMilli(last)); + Mockito.lenient().when(flow.getFirstSwitched()).thenReturn(Instant.ofEpochMilli(delta)); + Mockito.lenient().when(flow.getBytes()).thenReturn(bytes); + Mockito.lenient().when(flow.getDirection()).thenReturn(direction); + Mockito.lenient().when(flow.getApplication()).thenReturn(application); + Mockito.lenient().when(flow.getSrcAddr()).thenReturn(src); + Mockito.lenient().when(flow.getDstAddr()).thenReturn(dst); + Mockito.lenient().when(flow.getSamplingInterval()).thenReturn(1.0); + return flow; + } + + private static void assertArrayEquals2(long[] expected, long[] actual) { + Assert.assertTrue("expected " + Arrays.toString(expected) + " but was " + Arrays.toString(actual), + actual != null && expected.length == actual.length && expected[0] == actual[0] && expected[1] == actual[1]); + } +} \ No newline at end of file diff --git a/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/ProportionalSumParityTest.java b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/ProportionalSumParityTest.java new file mode 100644 index 000000000000..68cfc25bcd60 --- /dev/null +++ b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/ProportionalSumParityTest.java @@ -0,0 +1,245 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.postgres; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Random; +import java.util.TreeSet; + +import org.junit.Assert; +import org.junit.Test; + +/** + * Verifies that {@link FlowProration} (the pure-Java mirror of the proration SQL rendered by + * {@link PostgresFlowQueryService}) produces byte-for-byte the same results as a faithful port of + * the Elasticsearch drift-plugin's {@code ProportionalSumAggregator}. The {@code drift*} helpers + * below are that port; the fuzz tests exercise hundreds of thousands of random flow/window/step + * combinations (including non-zero histogram offsets) to lock the two arithmetic paths together. + */ +public class ProportionalSumParityTest { + private static double tol(double expected) { + return Math.max(1.0E-6, Math.abs(expected) * 1.0E-9); + } + + private static long roundDown(long v, long step) { + return Math.floorDiv(v, step) * step; + } + + private static long timeInWindow(long ws, long we, long rs, long re) { + if (rs > we || re < ws) { + return 0L; + } + return Math.min(we, re) - Math.max(ws, rs); + } + + private static Map drift(long bytes, Double sampling, long delta, long last, long queryStart, long queryEnd, long step, long offset) { + double value = bytes; + if (sampling != null && Double.isFinite(sampling) && sampling != 0.0) { + value *= sampling.doubleValue(); + } + long rangeStart = delta; + long rangeEnd = last; + long rangeDuration = rangeEnd - rangeStart; + long startRounded = ProportionalSumParityTest.roundDown(Math.max(rangeStart, queryStart) - offset, step) + offset; + long lastRounded = ProportionalSumParityTest.roundDown(Math.min(rangeEnd, queryEnd) - offset, step) + offset; + LinkedHashMap buckets = new LinkedHashMap(); + long bucketStart = startRounded; + while (bucketStart <= lastRounded) { + long nextBucketStart = bucketStart + step; + long timeInBucket = ProportionalSumParityTest.timeInWindow(bucketStart, nextBucketStart, rangeStart, rangeEnd); + double ratio = rangeDuration != 0L ? (double)timeInBucket / (double)rangeDuration : 1.0; + buckets.merge(bucketStart, value * ratio, Double::sum); + bucketStart = nextBucketStart; + } + return buckets; + } + + private static double driftSummary(long bytes, Double sampling, long delta, long last, long start, long end) { + long step = end - start; + double sum = 0.0; + for (double v : ProportionalSumParityTest.drift(bytes, sampling, delta, last, start, end - 1L, step, start).values()) { + sum += v; + } + return sum; + } + + private static Map driftSeries(long bytes, Double sampling, long delta, long last, long start, long end, long step, long offset) { + return ProportionalSumParityTest.drift(bytes, sampling, delta, last, start, end, step, offset); + } + + @Test + public void summaryFlowFullyInsideWindowContributesWholeValue() { + ProportionalSumParityTest.assertSummaryParity(1000L, 1.0, 1200L, 1700L, 1000L, 2000L); + Assert.assertEquals(1000.0, FlowProration.summaryValue(1000L, 1.0, 1200L, 1700L, 1000L, 2000L), ProportionalSumParityTest.tol(1000.0)); + } + + @Test + public void summaryFlowStraddlingStartContributesInWindowFraction() { + ProportionalSumParityTest.assertSummaryParity(1000L, 1.0, 800L, 1200L, 1000L, 2000L); + Assert.assertEquals(500.0, FlowProration.summaryValue(1000L, 1.0, 800L, 1200L, 1000L, 2000L), ProportionalSumParityTest.tol(500.0)); + } + + @Test + public void summaryFlowStraddlingEndContributesInWindowFraction() { + ProportionalSumParityTest.assertSummaryParity(1000L, 1.0, 1800L, 2200L, 1000L, 2000L); + Assert.assertEquals(500.0, FlowProration.summaryValue(1000L, 1.0, 1800L, 2200L, 1000L, 2000L), ProportionalSumParityTest.tol(500.0)); + } + + @Test + public void summaryZeroDurationInsideWindowContributesWholeValue() { + ProportionalSumParityTest.assertSummaryParity(1000L, 1.0, 1500L, 1500L, 1000L, 2000L); + Assert.assertEquals(1000.0, FlowProration.summaryValue(1000L, 1.0, 1500L, 1500L, 1000L, 2000L), ProportionalSumParityTest.tol(1000.0)); + } + + @Test + public void summaryZeroDurationAtExactEndIsExcluded() { + ProportionalSumParityTest.assertSummaryParity(1000L, 1.0, 2000L, 2000L, 1000L, 2000L); + Assert.assertEquals(0.0, FlowProration.summaryValue(1000L, 1.0, 2000L, 2000L, 1000L, 2000L), ProportionalSumParityTest.tol(0.0)); + } + + @Test + public void samplingIntervalScalesValueButZeroAndNonFiniteDoNot() { + Assert.assertEquals(4000.0, FlowProration.summaryValue(1000L, 4.0, 1200L, 1700L, 1000L, 2000L), ProportionalSumParityTest.tol(4000.0)); + Assert.assertEquals(1000.0, FlowProration.summaryValue(1000L, 0.0, 1200L, 1700L, 1000L, 2000L), ProportionalSumParityTest.tol(1000.0)); + Assert.assertEquals(1000.0, FlowProration.summaryValue(1000L, Double.NaN, 1200L, 1700L, 1000L, 2000L), ProportionalSumParityTest.tol(1000.0)); + Assert.assertEquals(1000.0, FlowProration.summaryValue(1000L, Double.POSITIVE_INFINITY, 1200L, 1700L, 1000L, 2000L), ProportionalSumParityTest.tol(1000.0)); + Assert.assertEquals(1000.0, FlowProration.summaryValue(1000L, null, 1200L, 1700L, 1000L, 2000L), ProportionalSumParityTest.tol(1000.0)); + ProportionalSumParityTest.assertSummaryParity(1000L, 4.0, 1200L, 1700L, 1000L, 2000L); + ProportionalSumParityTest.assertSummaryParity(1000L, 0.0, 1200L, 1700L, 1000L, 2000L); + } + + @Test + public void seriesDistributesAcrossBucketsProportionally() { + ProportionalSumParityTest.assertSeriesParity(1000L, 1.0, 1000L, 2000L, 1000L, 2000L, 250L, 0L); + Map s = FlowProration.series(1000L, 1.0, 1000L, 2000L, 1000L, 2000L, 250L); + Assert.assertEquals(250.0, s.get(1000L), ProportionalSumParityTest.tol(250.0)); + Assert.assertEquals(250.0, s.get(1250L), ProportionalSumParityTest.tol(250.0)); + Assert.assertEquals(250.0, s.get(1500L), ProportionalSumParityTest.tol(250.0)); + Assert.assertEquals(250.0, s.get(1750L), ProportionalSumParityTest.tol(250.0)); + } + + @Test + public void seriesZeroDurationLandsWholeValueInOneBucket() { + ProportionalSumParityTest.assertSeriesParity(1000L, 1.0, 1500L, 1500L, 1000L, 2000L, 250L, 0L); + Map s = FlowProration.series(1000L, 1.0, 1500L, 1500L, 1000L, 2000L, 250L); + Assert.assertEquals(1000.0, s.get(1500L), ProportionalSumParityTest.tol(1000.0)); + } + + @Test + public void fuzzSummaryParity() { + Random rnd = new Random(4044185325L); + for (int i = 0; i < 200000; ++i) { + long start = 1000000L + (long)rnd.nextInt(1000000); + long end = start + 1L + (long)rnd.nextInt(2000000); + long[] range = ProportionalSumParityTest.randomRange(rnd, start, end); + long bytes = 1 + rnd.nextInt(1000000); + Double sampling = ProportionalSumParityTest.randomSampling(rnd); + ProportionalSumParityTest.assertSummaryParity(bytes, sampling, range[0], range[1], start, end); + } + } + + @Test + public void fuzzSeriesParity() { + Random rnd = new Random(6169061L); + for (int i = 0; i < 100000; ++i) { + long start = 1000000L + (long)rnd.nextInt(1000000); + long span = 1 + rnd.nextInt(2000000); + long end = start + span; + long step = 1 + rnd.nextInt((int)Math.min(span, 500000L)); + long[] range = ProportionalSumParityTest.randomRange(rnd, start, end); + long bytes = 1 + rnd.nextInt(1000000); + Double sampling = ProportionalSumParityTest.randomSampling(rnd); + ProportionalSumParityTest.assertSeriesParity(bytes, sampling, range[0], range[1], start, end, step, 0L); + } + } + + @Test + public void fuzzSeriesParityWithNonZeroOffset() { + Random rnd = new Random(1045991L); + for (int i = 0; i < 100000; ++i) { + long start = 1000000L + (long)rnd.nextInt(1000000); + long span = 1 + rnd.nextInt(2000000); + long end = start + span; + long step = 1 + rnd.nextInt((int)Math.min(span, 500000L)); + long origin = rnd.nextInt((int)step); + long[] range = ProportionalSumParityTest.randomRange(rnd, start, end); + long bytes = 1 + rnd.nextInt(1000000); + Double sampling = ProportionalSumParityTest.randomSampling(rnd); + ProportionalSumParityTest.assertSeriesParity(bytes, sampling, range[0], range[1], start, end, step, origin); + } + } + + private static long[] randomRange(Random rnd, long start, long end) { + long last; + long lo = start - 500000L; + long span = end - start + 1000000L; + long delta = lo + (long)(rnd.nextDouble() * (double)span); + long l = last = rnd.nextInt(5) == 0 ? delta : lo + (long)(rnd.nextDouble() * (double)span); + if (last < delta) { + long t = delta; + delta = last; + last = t; + } + return new long[]{delta, last}; + } + + private static Double randomSampling(Random rnd) { + switch (rnd.nextInt(6)) { + case 0: { + return null; + } + case 1: { + return 0.0; + } + case 2: { + return Double.NaN; + } + case 3: { + return Double.POSITIVE_INFINITY; + } + case 4: { + return 1.0; + } + } + return 1.0 + (double)rnd.nextInt(1000); + } + + private static void assertSummaryParity(long bytes, Double sampling, long delta, long last, long start, long end) { + double expected = ProportionalSumParityTest.driftSummary(bytes, sampling, delta, last, start, end); + double actual = FlowProration.summaryValue(bytes, sampling, delta, last, start, end); + Assert.assertEquals("summary bytes=" + bytes + " sampling=" + sampling + " delta=" + delta + " last=" + last + " window=[" + start + "," + end + ")", expected, actual, ProportionalSumParityTest.tol(expected)); + } + + private static void assertSeriesParity(long bytes, Double sampling, long delta, long last, long start, long end, long step, long origin) { + Map expected = ProportionalSumParityTest.driftSeries(bytes, sampling, delta, last, start, end, step, origin); + Map actual = FlowProration.series(bytes, sampling, delta, last, start, end, step, origin); + TreeSet keys = new TreeSet(); + keys.addAll(expected.keySet()); + keys.addAll(actual.keySet()); + for (Long k : keys) { + double ev = expected.getOrDefault(k, 0.0); + double av = actual.getOrDefault(k, 0.0); + Assert.assertEquals("series bucket=" + k + " bytes=" + bytes + " sampling=" + sampling + " delta=" + delta + " last=" + last + " window=[" + start + "," + end + "] step=" + step, ev, av, ProportionalSumParityTest.tol(ev)); + } + } +} \ No newline at end of file diff --git a/opennms-base-assembly/src/main/filtered/etc/vacuumd-configuration.xml b/opennms-base-assembly/src/main/filtered/etc/vacuumd-configuration.xml index 465906982a5e..908b7bb65383 100644 --- a/opennms-base-assembly/src/main/filtered/etc/vacuumd-configuration.xml +++ b/opennms-base-assembly/src/main/filtered/etc/vacuumd-configuration.xml @@ -4,6 +4,14 @@ DELETE FROM snmpInterface WHERE snmpInterface.snmpCollect = 'D'; + + + DO $$ BEGIN IF EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'flow_maintain_partitions') THEN PERFORM flow_maintain_partitions(14, 3); END IF; END $$; + + DELETE FROM node WHERE node.nodeType = 'D'; diff --git a/opennms-full-assembly/pom.xml b/opennms-full-assembly/pom.xml index 58555d68f507..068e9ca875e0 100644 --- a/opennms-full-assembly/pom.xml +++ b/opennms-full-assembly/pom.xml @@ -504,6 +504,7 @@ opennms-es-rest ifttt-integration opennms-flows + opennms-flows-postgres opennms-api-layer opennms-plugin-sample karaf-extender From 5dadd1ff8218299201bc63547ac75d08e9b72d82 Mon Sep 17 00:00:00 2001 From: Dino Date: Wed, 8 Jul 2026 14:40:24 -0500 Subject: [PATCH 02/15] Do lots of OSGi BS so the feature loads cleanly and overrides Elastic flow persistence, while still allowing Kafka persistence to forward if configured; make everything coexist --- .../OSGI-INF/blueprint/blueprint.xml | 3 + .../OSGI-INF/blueprint/blueprint.xml | 16 +- .../flows/processing/impl/PipelineImpl.java | 69 ++++++- .../impl/PipelineImplRankingTest.java | 147 ++++++++++++++ features/flows/rest/impl/pom.xml | 6 + .../HighestRankedFlowQueryService.java | 183 ++++++++++++++++++ .../OSGI-INF/blueprint/blueprint.xml | 8 +- 7 files changed, 420 insertions(+), 12 deletions(-) create mode 100644 features/flows/processing/src/test/java/org/opennms/netmgt/flows/processing/impl/PipelineImplRankingTest.java create mode 100644 features/flows/rest/impl/src/main/java/org/opennms/netmgt/flows/rest/internal/HighestRankedFlowQueryService.java diff --git a/features/flows/kafka-persistence/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/features/flows/kafka-persistence/src/main/resources/OSGI-INF/blueprint/blueprint.xml index af096daaeee7..04e2ef2203d0 100644 --- a/features/flows/kafka-persistence/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ b/features/flows/kafka-persistence/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@ -67,6 +67,9 @@ + + diff --git a/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml index 68b7fb9a6bfc..fd5f863c8631 100644 --- a/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ b/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@ -35,16 +35,20 @@ - - + + - + @@ -52,7 +56,7 @@ - + diff --git a/features/flows/processing/src/main/java/org/opennms/netmgt/flows/processing/impl/PipelineImpl.java b/features/flows/processing/src/main/java/org/opennms/netmgt/flows/processing/impl/PipelineImpl.java index cc420755f401..664d2966bc15 100644 --- a/features/flows/processing/src/main/java/org/opennms/netmgt/flows/processing/impl/PipelineImpl.java +++ b/features/flows/processing/src/main/java/org/opennms/netmgt/flows/processing/impl/PipelineImpl.java @@ -48,6 +48,16 @@ public class PipelineImpl implements Pipeline { public static final String REPOSITORY_ID = "flows.repository.id"; + /** Standard OSGi service property; a higher value overrides lower-ranked primary flow stores. */ + private static final String SERVICE_RANKING = "service.ranking"; + + /** + * Service property marking a repository as a forwarder (e.g. Kafka) rather than a primary store + * (e.g. Elasticsearch, PostgreSQL). Forwarders always receive flows and are exempt from the + * highest-ranked-store override, so they coexist with whichever primary store is active. + */ + private static final String FORWARDER = "flows.repository.forwarder"; + private static final Logger LOG = LoggerFactory.getLogger(PipelineImpl.class); /** @@ -135,9 +145,28 @@ public void process(final List flows, final FlowSource source, final Proce throw new FlowException("Failed to threshold one or more flows.", e); } - // Push flows to persistence - for (final var persister : this.persisters.entrySet()) { - persister.getValue().persist(enrichedFlows); + // Push flows to persistence. Forwarders (e.g. Kafka) always receive the flows. Among primary + // stores (e.g. Elasticsearch, PostgreSQL) only the highest service-ranked one does, so a + // higher-ranked store (the opennms-flows-postgres feature at ranking 100) overrides lower-ranked + // stores (the default Elasticsearch repository at ranking 0) entirely — no dual-write, no + // operator configuration — while forwarders keep running alongside it. Each persist is isolated + // so one repository's failure cannot starve the others. + final int maxStoreRanking = this.persisters.values().stream() + .filter(p -> !p.forwarder) + .mapToInt(p -> p.ranking) + .max() + .orElse(Integer.MIN_VALUE); + for (final var entry : this.persisters.entrySet()) { + final Persister persister = entry.getValue(); + if (!persister.forwarder && persister.ranking != maxStoreRanking) { + continue; // a lower-ranked primary store, overridden by a higher-ranked one + } + try { + persister.persist(enrichedFlows); + } catch (final Exception e) { + LOG.error("Flow repository '{}' failed to persist {} flows; continuing with other repositories.", + entry.getKey(), enrichedFlows.size(), e); + } } } @@ -149,8 +178,34 @@ public synchronized void onBind(final FlowRepository repository, final Map prope } final String pid = Objects.toString(properties.get(REPOSITORY_ID)); - this.persisters.put(pid, new Persister(repository, + final int ranking = rankingOf(properties); + final boolean forwarder = forwarderOf(properties); + this.persisters.put(pid, new Persister(repository, ranking, forwarder, this.metricRegistry.timer(MetricRegistry.name("logPersisting", pid)))); + LOG.info("Bound flow repository '{}' (ranking {}, forwarder {}).", pid, ranking, forwarder); + } + + private static int rankingOf(@SuppressWarnings("rawtypes") final Map properties) { + final Object ranking = properties.get(SERVICE_RANKING); + if (ranking instanceof Number) { + return ((Number) ranking).intValue(); + } + if (ranking != null) { + try { + return Integer.parseInt(ranking.toString()); + } catch (final NumberFormatException ignored) { + // fall through to default + } + } + return 0; + } + + private static boolean forwarderOf(@SuppressWarnings("rawtypes") final Map properties) { + final Object forwarder = properties.get(FORWARDER); + if (forwarder instanceof Boolean) { + return (Boolean) forwarder; + } + return forwarder != null && Boolean.parseBoolean(forwarder.toString()); } @SuppressWarnings("rawtypes") @@ -166,10 +221,14 @@ public synchronized void onUnbind(final FlowRepository repository, final Map pro private static class Persister { public final FlowRepository repository; + public final int ranking; + public final boolean forwarder; public final Timer logTimer; - public Persister(final FlowRepository repository, final Timer logTimer) { + public Persister(final FlowRepository repository, final int ranking, final boolean forwarder, final Timer logTimer) { this.repository = Objects.requireNonNull(repository); + this.ranking = ranking; + this.forwarder = forwarder; this.logTimer = Objects.requireNonNull(logTimer); } diff --git a/features/flows/processing/src/test/java/org/opennms/netmgt/flows/processing/impl/PipelineImplRankingTest.java b/features/flows/processing/src/test/java/org/opennms/netmgt/flows/processing/impl/PipelineImplRankingTest.java new file mode 100644 index 000000000000..583cf1c99102 --- /dev/null +++ b/features/flows/processing/src/test/java/org/opennms/netmgt/flows/processing/impl/PipelineImplRankingTest.java @@ -0,0 +1,147 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.processing.impl; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.junit.Before; +import org.junit.Test; +import org.opennms.integration.api.v1.flows.FlowException; +import org.opennms.integration.api.v1.flows.FlowRepository; +import org.opennms.netmgt.flows.api.Flow; +import org.opennms.netmgt.flows.api.FlowSource; + +import com.codahale.metrics.MetricRegistry; + +/** + * Verifies the pipeline's flow-repository routing: only the highest service-ranked tier of repositories + * receives flows (so the higher-ranked opennms-flows-postgres feature overrides the default Elasticsearch + * repository entirely, and uninstalling it falls back), and a failing repository does not starve others. + */ +public class PipelineImplRankingTest { + + private static final String RANKING = "service.ranking"; + private static final String FORWARDER = "flows.repository.forwarder"; + + private PipelineImpl pipeline; + + @Before + public void setUp() { + final DocumentEnricherImpl enricher = mock(DocumentEnricherImpl.class); + final InterfaceMarkerImpl marker = mock(InterfaceMarkerImpl.class); + final FlowThresholdingImpl thresholding = mock(FlowThresholdingImpl.class); + when(enricher.enrich(any(), any())).thenReturn(Collections.emptyList()); + pipeline = new PipelineImpl(new MetricRegistry(), enricher, marker, thresholding); + } + + private void process() throws Exception { + pipeline.process(List.of(mock(Flow.class)), mock(FlowSource.class), null); + } + + @Test + public void higherRankedRepositoryOverridesLowerRankedOnesEntirely() throws Exception { + final FlowRepository elastic = mock(FlowRepository.class); + final FlowRepository postgres = mock(FlowRepository.class); + pipeline.onBind(elastic, Map.of(PipelineImpl.REPOSITORY_ID, "elastic")); // ranking 0 + pipeline.onBind(postgres, Map.of(PipelineImpl.REPOSITORY_ID, "postgres", RANKING, 100)); // ranking 100 + + process(); + + verify(postgres, times(1)).persist(any()); + verify(elastic, never()).persist(any()); + } + + @Test + public void fallsBackToLowerRankedRepositoryWhenTopRankedUnbinds() throws Exception { + final FlowRepository elastic = mock(FlowRepository.class); + final FlowRepository postgres = mock(FlowRepository.class); + pipeline.onBind(elastic, Map.of(PipelineImpl.REPOSITORY_ID, "elastic")); + pipeline.onBind(postgres, Map.of(PipelineImpl.REPOSITORY_ID, "postgres", RANKING, 100)); + + pipeline.onUnbind(postgres, Map.of(PipelineImpl.REPOSITORY_ID, "postgres", RANKING, 100)); + process(); + + verify(elastic, times(1)).persist(any()); + } + + @Test + public void allRepositoriesInTheTopTierAreWritten() throws Exception { + final FlowRepository a = mock(FlowRepository.class); + final FlowRepository b = mock(FlowRepository.class); + pipeline.onBind(a, Map.of(PipelineImpl.REPOSITORY_ID, "a")); // both default ranking 0 + pipeline.onBind(b, Map.of(PipelineImpl.REPOSITORY_ID, "b")); + + process(); + + verify(a, times(1)).persist(any()); + verify(b, times(1)).persist(any()); + } + + @Test + public void forwarderCoexistsWithTheTopRankedStore() throws Exception { + final FlowRepository elastic = mock(FlowRepository.class); + final FlowRepository postgres = mock(FlowRepository.class); + final FlowRepository kafka = mock(FlowRepository.class); + pipeline.onBind(elastic, Map.of(PipelineImpl.REPOSITORY_ID, "elastic")); // store, ranking 0 + pipeline.onBind(postgres, Map.of(PipelineImpl.REPOSITORY_ID, "postgres", RANKING, 100)); // store, ranking 100 + pipeline.onBind(kafka, Map.of(PipelineImpl.REPOSITORY_ID, "kafka", FORWARDER, true)); // forwarder + + process(); + + verify(postgres, times(1)).persist(any()); // top-ranked store wins + verify(kafka, times(1)).persist(any()); // forwarder always runs alongside + verify(elastic, never()).persist(any()); // overridden store + } + + @Test + public void forwarderReceivesFlowsEvenWithNoStoreSelected() throws Exception { + final FlowRepository kafka = mock(FlowRepository.class); + pipeline.onBind(kafka, Map.of(PipelineImpl.REPOSITORY_ID, "kafka", FORWARDER, true)); + + process(); + + verify(kafka, times(1)).persist(any()); + } + + @Test + public void aFailingRepositoryDoesNotStarveOthersInTheSameTier() throws Exception { + final FlowRepository bad = mock(FlowRepository.class); + final FlowRepository good = mock(FlowRepository.class); + doThrow(new FlowException("boom")).when(bad).persist(any()); + pipeline.onBind(bad, Map.of(PipelineImpl.REPOSITORY_ID, "bad")); + pipeline.onBind(good, Map.of(PipelineImpl.REPOSITORY_ID, "good")); + + process(); // must not propagate the failure + + verify(good, times(1)).persist(any()); + } +} diff --git a/features/flows/rest/impl/pom.xml b/features/flows/rest/impl/pom.xml index 848133f123b2..1ca343ed555e 100644 --- a/features/flows/rest/impl/pom.xml +++ b/features/flows/rest/impl/pom.xml @@ -16,6 +16,12 @@ org.opennms.features.flows.rest.api ${project.version} + + + org.osgi + osgi.core + provided + org.opennms.core org.opennms.core.web diff --git a/features/flows/rest/impl/src/main/java/org/opennms/netmgt/flows/rest/internal/HighestRankedFlowQueryService.java b/features/flows/rest/impl/src/main/java/org/opennms/netmgt/flows/rest/internal/HighestRankedFlowQueryService.java new file mode 100644 index 000000000000..a59812454b05 --- /dev/null +++ b/features/flows/rest/impl/src/main/java/org/opennms/netmgt/flows/rest/internal/HighestRankedFlowQueryService.java @@ -0,0 +1,183 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.rest.internal; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +import org.opennms.netmgt.flows.api.Conversation; +import org.opennms.netmgt.flows.api.Directional; +import org.opennms.netmgt.flows.api.FlowQueryService; +import org.opennms.netmgt.flows.api.Host; +import org.opennms.netmgt.flows.api.LimitedCardinalityField; +import org.opennms.netmgt.flows.api.TrafficSummary; +import org.opennms.netmgt.flows.filter.api.Filter; +import org.osgi.framework.BundleContext; +import org.osgi.util.tracker.ServiceTracker; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Table; + +/** + * A {@link FlowQueryService} that always delegates to the highest service-ranked {@code FlowQueryService} + * currently registered, re-selecting live as services come and go. + * + *

OpenNMS ships Elasticsearch flow reads ({@code SmartQueryService}, default ranking) as part of the + * {@code opennms-flows} feature. The optional {@code opennms-flows-postgres} feature registers its + * {@code PostgresFlowQueryService} at a higher ranking, so simply installing that feature forces flow + * reads onto PostgreSQL with no operator configuration; uninstalling it falls back to Elasticsearch. + * A plain Blueprint {@code } could not do this — it binds the best service available at bind + * time and never swaps to a higher-ranked one that appears later — so we track the services directly. + */ +public class HighestRankedFlowQueryService implements FlowQueryService { + + private static final Logger LOG = LoggerFactory.getLogger(HighestRankedFlowQueryService.class); + + private final BundleContext bundleContext; + private ServiceTracker tracker; + + public HighestRankedFlowQueryService(final BundleContext bundleContext) { + this.bundleContext = bundleContext; + } + + public void init() { + tracker = new ServiceTracker<>(bundleContext, FlowQueryService.class, null); + tracker.open(); + LOG.debug("HighestRankedFlowQueryService started; tracking FlowQueryService registrations."); + } + + public void destroy() { + if (tracker != null) { + tracker.close(); + tracker = null; + } + } + + /** The highest service-ranked FlowQueryService currently registered (ServiceTracker resolves the ranking). */ + private FlowQueryService delegate() { + final ServiceTracker t = tracker; + final FlowQueryService svc = (t != null) ? t.getService() : null; + if (svc == null) { + throw new IllegalStateException("No FlowQueryService is currently available."); + } + return svc; + } + + @Override + public CompletableFuture getFlowCount(final List filters) { + return delegate().getFlowCount(filters); + } + + @Override + public CompletableFuture> getApplications(final String matchingPrefix, final long limit, final List filters) { + return delegate().getApplications(matchingPrefix, limit, filters); + } + + @Override + public CompletableFuture>> getTopNApplicationSummaries(final int N, final boolean includeOther, final List filters) { + return delegate().getTopNApplicationSummaries(N, includeOther, filters); + } + + @Override + public CompletableFuture>> getApplicationSummaries(final Set applications, final boolean includeOther, final List filters) { + return delegate().getApplicationSummaries(applications, includeOther, filters); + } + + @Override + public CompletableFuture, Long, Double>> getApplicationSeries(final Set applications, final long step, final boolean includeOther, final List filters) { + return delegate().getApplicationSeries(applications, step, includeOther, filters); + } + + @Override + public CompletableFuture, Long, Double>> getTopNApplicationSeries(final int N, final long step, final boolean includeOther, final List filters) { + return delegate().getTopNApplicationSeries(N, step, includeOther, filters); + } + + @Override + public CompletableFuture> getConversations(final String locationPattern, final String protocolPattern, + final String lowerIPPattern, final String upperIPPattern, + final String applicationPattern, final long limit, final List filters) { + return delegate().getConversations(locationPattern, protocolPattern, lowerIPPattern, upperIPPattern, applicationPattern, limit, filters); + } + + @Override + public CompletableFuture>> getTopNConversationSummaries(final int N, final boolean includeOther, final List filters) { + return delegate().getTopNConversationSummaries(N, includeOther, filters); + } + + @Override + public CompletableFuture>> getConversationSummaries(final Set conversations, final boolean includeOther, final List filters) { + return delegate().getConversationSummaries(conversations, includeOther, filters); + } + + @Override + public CompletableFuture, Long, Double>> getConversationSeries(final Set conversations, final long step, final boolean includeOther, final List filters) { + return delegate().getConversationSeries(conversations, step, includeOther, filters); + } + + @Override + public CompletableFuture, Long, Double>> getTopNConversationSeries(final int N, final long step, final boolean includeOther, final List filters) { + return delegate().getTopNConversationSeries(N, step, includeOther, filters); + } + + @Override + public CompletableFuture> getHosts(final String regex, final long limit, final List filters) { + return delegate().getHosts(regex, limit, filters); + } + + @Override + public CompletableFuture>> getTopNHostSummaries(final int N, final boolean includeOther, final List filters) { + return delegate().getTopNHostSummaries(N, includeOther, filters); + } + + @Override + public CompletableFuture>> getHostSummaries(final Set hosts, final boolean includeOther, final List filters) { + return delegate().getHostSummaries(hosts, includeOther, filters); + } + + @Override + public CompletableFuture, Long, Double>> getHostSeries(final Set hosts, final long step, final boolean includeOther, final List filters) { + return delegate().getHostSeries(hosts, step, includeOther, filters); + } + + @Override + public CompletableFuture, Long, Double>> getTopNHostSeries(final int N, final long step, final boolean includeOther, final List filters) { + return delegate().getTopNHostSeries(N, step, includeOther, filters); + } + + @Override + public CompletableFuture> getFieldValues(final LimitedCardinalityField field, final List filters) { + return delegate().getFieldValues(field, filters); + } + + @Override + public CompletableFuture>> getFieldSummaries(final LimitedCardinalityField field, final List filters) { + return delegate().getFieldSummaries(field, filters); + } + + @Override + public CompletableFuture, Long, Double>> getFieldSeries(final LimitedCardinalityField field, final long step, final List filters) { + return delegate().getFieldSeries(field, step, filters); + } +} \ No newline at end of file diff --git a/features/flows/rest/impl/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/features/flows/rest/impl/src/main/resources/OSGI-INF/blueprint/blueprint.xml index 80393630b549..0d5e3f4026b6 100644 --- a/features/flows/rest/impl/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ b/features/flows/rest/impl/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@ -9,7 +9,13 @@ http://aries.apache.org/schemas/blueprint-cm/blueprint-cm-1.3.0.xsd "> - + + + + From 98fbb8547c2d10ce9d2bf89db1fe0ae831f28b3d Mon Sep 17 00:00:00 2001 From: Dino Date: Wed, 8 Jul 2026 21:38:13 -0500 Subject: [PATCH 03/15] Sentinel feature w/ external database config --- .../src/main/resources/features-sentinel.xml | 15 +++ .../features/src/main/resources/features.xml | 4 +- features/container/sentinel/pom.xml | 1 + .../postgres/FlowDataSourceProvider.java | 118 ++++++++++++++++++ .../postgres/PostgresFlowQueryService.java | 16 +-- .../postgres/PostgresFlowRepository.java | 17 +-- .../OSGI-INF/blueprint/blueprint.xml | 41 +++++- .../postgres/PostgresFlowRepositoryIT.java | 26 ++++ 8 files changed, 211 insertions(+), 27 deletions(-) create mode 100644 features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowDataSourceProvider.java diff --git a/container/features/src/main/resources/features-sentinel.xml b/container/features/src/main/resources/features-sentinel.xml index 5fce658354f7..a889d1532d10 100644 --- a/container/features/src/main/resources/features-sentinel.xml +++ b/container/features/src/main/resources/features-sentinel.xml @@ -160,6 +160,21 @@ mvn:org.opennms.features.distributed/org.opennms.features.distributed.collection/${project.version} + +

Optional PostgreSQL-backed flow persistence for Sentinel (write path). When installed it registers a + higher-ranked FlowRepository that overrides Elasticsearch persistence on this Sentinel; flow reads are + served by Horizon against the same database. Configure the flow database via the + org.opennms.features.flows.persistence.postgres pid (datasource.url/username/password/databaseName).
+ sentinel-flows + spring-jdbc + + opennms-core-db + mvn:com.fasterxml.jackson.core/jackson-databind/${jackson2Version} + mvn:org.yaml/snakeyaml/1.31 + mvn:org.liquibase/liquibase-core/${liquibaseVersion} + mvn:org.opennms.features.flows/org.opennms.features.flows.postgres/${project.version} + + sentinel-telemetry opennms-telemetry-nxos diff --git a/container/features/src/main/resources/features.xml b/container/features/src/main/resources/features.xml index cae2a192d9ad..fda50b5e105f 100644 --- a/container/features/src/main/resources/features.xml +++ b/container/features/src/main/resources/features.xml @@ -1218,7 +1218,9 @@
Optional PostgreSQL-backed FlowRepository/FlowQueryService (jsonb). Install alongside or instead of the Elasticsearch flow persistence.
opennms-flows spring-jdbc - postgresql + + opennms-core-db mvn:com.fasterxml.jackson.core/jackson-databind/${jackson2Version} + + + + + + + + + + + + @@ -24,11 +40,30 @@ + + + + + + + + + + + + + + + + - + @@ -52,7 +87,7 @@ - + diff --git a/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepositoryIT.java b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepositoryIT.java index 87e0a73dd46c..5e49e86f23af 100644 --- a/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepositoryIT.java +++ b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepositoryIT.java @@ -65,6 +65,9 @@ public class PostgresFlowRepositoryIT { private static final long WINDOW = 1000L; private static PostgreSQLContainer POSTGRES; private static PGSimpleDataSource DATA_SOURCE; + private static String JDBC_URL; + private static String JDBC_USER; + private static String JDBC_PASSWORD; private static PostgresFlowRepository repository; private static PostgresFlowQueryService queryService; private static JdbcTemplate jdbc; @@ -87,6 +90,9 @@ public static void startContainer() throws Exception { user = POSTGRES.getUsername(); password = POSTGRES.getPassword(); } + JDBC_URL = url; + JDBC_USER = user; + JDBC_PASSWORD = password; DATA_SOURCE = new PGSimpleDataSource(); DATA_SOURCE.setUrl(url); DATA_SOURCE.setUser(user); @@ -273,6 +279,26 @@ public void persistWritesRowsThroughTheBatchingWriter() throws Exception { Assert.assertEquals(6912L, total); } + @Test + public void dedicatedPoolProviderConnectsAndQueries() throws Exception { + final FlowDataSourceProvider provider = new FlowDataSourceProvider(); + provider.setUrl(JDBC_URL); + provider.setUsername(JDBC_USER); + provider.setPassword(JDBC_PASSWORD); + provider.setMinPool(1); + provider.setMaxPool(2); + provider.setMaxSize(2); + provider.init(); + try { + final DataSource ds = provider.getDataSource(); + Assert.assertNotNull("dedicated mode must build a pool when a url is set", ds); + final Integer one = new JdbcTemplate(ds).queryForObject("SELECT 1", Integer.class); + Assert.assertEquals(Integer.valueOf(1), one); + } finally { + provider.close(); + } + } + private static Flow mockFlow(String application, Flow.Direction direction, long delta, long last, long bytes, String src, String dst) { Flow flow = Mockito.mock(Flow.class); Mockito.lenient().when(flow.getTimestamp()).thenReturn(Instant.ofEpochMilli(last)); From 44f3352b1134fe209306324b57388848bbfd8669 Mon Sep 17 00:00:00 2001 From: Dino Date: Thu, 9 Jul 2026 13:15:24 -0500 Subject: [PATCH 04/15] NMS-19978: ship stax2-api to satisfy aalto-xml's OSGi import in opennms-core-camel --- container/features/src/main/resources/features.xml | 2 ++ pom.xml | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/container/features/src/main/resources/features.xml b/container/features/src/main/resources/features.xml index fda50b5e105f..689820955691 100644 --- a/container/features/src/main/resources/features.xml +++ b/container/features/src/main/resources/features.xml @@ -342,6 +342,8 @@ mvn:io.netty/netty-transport-classes-epoll/${netty4Version} camel-netty4 opennms-core + + mvn:org.codehaus.woodstox/stax2-api/${stax2ApiVersion} mvn:com.fasterxml/aalto-xml/${aaltoXmlVersion} mvn:io.netty/netty-codec-xml/${netty4Version} mvn:org.opennms.core/org.opennms.core.camel/${project.version} diff --git a/pom.xml b/pom.xml index 0873ac612ea0..cc2959de9218 100644 --- a/pom.xml +++ b/pom.xml @@ -1736,6 +1736,7 @@ 1.3.3 + 4.2.2 1.10.13 2.33 1.1.3 @@ -5061,6 +5062,12 @@ aalto-xml ${aaltoXmlVersion} + + + org.codehaus.woodstox + stax2-api + ${stax2ApiVersion} + io.netty netty-all From 7619b4edb5ddd80ccbeea0b8f01ef0d88b1d4fe2 Mon Sep 17 00:00:00 2001 From: Dino Date: Thu, 9 Jul 2026 15:46:26 -0500 Subject: [PATCH 05/15] Clean up feature loading when the cnfiguration doesn't exist yet; behave damnit. --- .../postgres/FlowDataSourceProvider.java | 56 +++++++----- .../postgres/PostgresFlowQueryService.java | 28 ++++-- .../postgres/PostgresFlowRepository.java | 24 +++-- .../OSGI-INF/blueprint/blueprint.xml | 7 +- .../postgres/PostgresFlowDisabledTest.java | 87 +++++++++++++++++++ 5 files changed, 169 insertions(+), 33 deletions(-) create mode 100644 features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowDisabledTest.java diff --git a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowDataSourceProvider.java b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowDataSourceProvider.java index e155ef4f3b49..314f6378cd1e 100644 --- a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowDataSourceProvider.java +++ b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowDataSourceProvider.java @@ -66,31 +66,47 @@ public class FlowDataSourceProvider { /** Non-null only in dedicated mode; the pool this provider created and must close. */ private ClosableDataSource ownedPool; - public void init() throws Exception { + public void init() { + try { + this.dataSource = createDataSource(); + } catch (final Exception e) { + // Never fail the blueprint container over a missing/invalid flow datasource: load cleanly, + // log, and stay inert (getDataSource() == null) so the repository and query service disable + // themselves. With datasource.url blank the internal OpenNMS datasource is used, which only + // exists on Horizon; on Sentinel or for an external database, datasource.url (+ credentials) + // must be set on the org.opennms.features.flows.persistence.postgres pid. + this.dataSource = null; + LOG.error("PostgreSQL flow persistence is DISABLED: could not obtain a flow DataSource ({}). " + + "Set datasource.url/username/password on the org.opennms.features.flows.persistence.postgres " + + "pid (required on Sentinel and for external databases); with it blank the internal OpenNMS " + + "datasource is used, which is only available on Horizon.", e.toString()); + } + } + + private DataSource createDataSource() throws Exception { if (url == null || url.trim().isEmpty()) { DataSourceFactory.init(internalDataSourceName); - this.dataSource = DataSourceFactory.getInstance(internalDataSourceName); - LOG.info("PostgresFlow persistence using the internal '{}' datasource (no dedicated flow database " + LOG.info("PostgreSQL flow persistence using the internal '{}' datasource (no dedicated flow database " + "configured; set datasource.url on org.opennms.features.flows.persistence.postgres to use one).", internalDataSourceName); - } else { - final JdbcDataSource cfg = new JdbcDataSource(); - cfg.setName("opennms-flows"); - cfg.setDatabaseName(databaseName); - cfg.setClassName(driverClass); - cfg.setUrl(url); - cfg.setUserName(username); - cfg.setPassword(password); - final HikariCPConnectionFactory pool = new HikariCPConnectionFactory(cfg); - pool.setIdleTimeout(idleTimeout); - pool.setLoginTimeout(loginTimeout); - pool.setMinPool(minPool); - pool.setMaxPool(maxPool); - pool.setMaxSize(maxSize); - this.ownedPool = pool; - this.dataSource = pool; - LOG.info("PostgresFlow persistence using a dedicated connection pool for {}.", url); + return DataSourceFactory.getInstance(internalDataSourceName); } + final JdbcDataSource cfg = new JdbcDataSource(); + cfg.setName("opennms-flows"); + cfg.setDatabaseName(databaseName); + cfg.setClassName(driverClass); + cfg.setUrl(url); + cfg.setUserName(username); + cfg.setPassword(password); + final HikariCPConnectionFactory pool = new HikariCPConnectionFactory(cfg); + this.ownedPool = pool; // record early so close() can reclaim it even if a setter below fails + pool.setIdleTimeout(idleTimeout); + pool.setLoginTimeout(loginTimeout); + pool.setMinPool(minPool); + pool.setMaxPool(maxPool); + pool.setMaxSize(maxSize); + LOG.info("PostgreSQL flow persistence using a dedicated connection pool for {}.", url); + return pool; } public DataSource getDataSource() { diff --git a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresFlowQueryService.java b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresFlowQueryService.java index 6f8130baf273..c9232f74f33d 100644 --- a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresFlowQueryService.java +++ b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresFlowQueryService.java @@ -25,7 +25,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; @@ -105,14 +104,24 @@ private static String proratedForBucket(final long step) { private int threads = 4; + private FlowDataSourceProvider dataSourceProvider; private DataSource dataSource; private JdbcTemplate jdbcTemplate; private ExecutorService executor; public void start() { - final DataSource ds = Objects.requireNonNull(this.dataSource, "A DataSource must be set before start(); the " - + "blueprint injects a pooled DataSource built from the org.opennms.features.flows.persistence.postgres pid."); - this.jdbcTemplate = new JdbcTemplate(ds); + if (this.dataSource == null && this.dataSourceProvider != null) { + this.dataSource = this.dataSourceProvider.getDataSource(); + } + if (this.dataSource == null) { + // No flow DataSource configured (see FlowDataSourceProvider). Stay inert rather than failing + // the blueprint container: the feature loads; query methods complete exceptionally if called. + LOG.error("PostgresFlowQueryService not started: no flow DataSource is configured. Flow queries " + + "will NOT be served by PostgreSQL until datasource.url is set on the " + + "org.opennms.features.flows.persistence.postgres pid."); + return; + } + this.jdbcTemplate = new JdbcTemplate(this.dataSource); this.executor = Executors.newFixedThreadPool(threads, r -> { final Thread t = new Thread(r, "postgres-flow-query"); t.setDaemon(true); @@ -128,6 +137,13 @@ public void stop() { } private CompletableFuture async(final Supplier supplier) { + if (executor == null) { + // Inert (no DataSource configured): fail the query cleanly instead of NPEing. + final CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally(new IllegalStateException("PostgresFlowQueryService is not configured: " + + "no flow DataSource (set datasource.url on the org.opennms.features.flows.persistence.postgres pid).")); + return failed; + } return CompletableFuture.supplyAsync(supplier, executor); } @@ -683,6 +699,8 @@ private Table, Long, Double> mapSeries(final Table writer; @@ -83,8 +84,17 @@ public PostgresFlowRepository(final MetricRegistry metrics) { } public void start() throws Exception { - Objects.requireNonNull(this.dataSource, "A DataSource must be set before start(); the blueprint injects a " - + "pooled DataSource built from the org.opennms.features.flows.persistence.postgres pid."); + if (this.dataSource == null && this.dataSourceProvider != null) { + this.dataSource = this.dataSourceProvider.getDataSource(); + } + if (this.dataSource == null) { + // No flow DataSource was configured (see FlowDataSourceProvider). Stay inert rather than + // failing the blueprint container: the feature loads, persist() becomes a no-op. + LOG.error("PostgresFlowRepository not started: no flow DataSource is configured. Flows will NOT be " + + "persisted to PostgreSQL until datasource.url is set on the " + + "org.opennms.features.flows.persistence.postgres pid."); + return; + } this.jdbcTemplate = new JdbcTemplate(dataSource); if (runSchemaChangelog) { installSchema(); @@ -105,11 +115,13 @@ public void stop() { @Override public void persist(final Collection flows) throws FlowException { - if (flows == null || flows.isEmpty()) { + final BatchingFlowWriter w = this.writer; + if (w == null || flows == null || flows.isEmpty()) { + // Inert (no DataSource configured) or nothing to do. return; } for (final Flow flow : flows) { - writer.add(rowMapper.toRow(flow)); + w.add(rowMapper.toRow(flow)); } } @@ -167,7 +179,9 @@ private static void setInt(final PreparedStatement ps, final int idx, final Inte } // --- config setters (blueprint) --- - /** The dedicated flow-database DataSource (a pooled DataSource injected by the blueprint, or a test DataSource). */ + /** Blueprint wiring: supplies the flow DataSource in start() (may be inert, i.e. return null). */ + public void setDataSourceProvider(final FlowDataSourceProvider dataSourceProvider) { this.dataSourceProvider = dataSourceProvider; } + /** Test/embedding hook: use this DataSource directly instead of resolving one from the provider. */ public void setDataSource(final DataSource dataSource) { this.dataSource = dataSource; } public void setBatchSize(final int batchSize) { this.batchSize = batchSize; } public void setFlushIntervalMs(final long flushIntervalMs) { this.flushIntervalMs = flushIntervalMs; } diff --git a/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml index 7a82d03eff22..2bf5c034344d 100644 --- a/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ b/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@ -57,13 +57,14 @@ - - + + @@ -87,7 +88,7 @@ - + diff --git a/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowDisabledTest.java b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowDisabledTest.java new file mode 100644 index 000000000000..63042cf43ec8 --- /dev/null +++ b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowDisabledTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.postgres; + +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import org.junit.Test; +import org.opennms.integration.api.v1.flows.Flow; +import org.opennms.netmgt.flows.filter.api.Filter; +import org.opennms.netmgt.flows.filter.api.TimeRangeFilter; + +import com.codahale.metrics.MetricRegistry; + +/** + * Verifies that the PostgreSQL flow bundle degrades gracefully when no usable flow DataSource is + * configured: the blueprint beans must load without throwing (so the feature installs cleanly), log an + * error, and then do nothing — rather than failing the blueprint container (which previously surfaced as + * an NPE when the internal datasource was unavailable, e.g. on Sentinel with an empty configuration). + */ +public class PostgresFlowDisabledTest { + + @Test + public void providerLoadsCleanlyAndStaysInertWhenTheDataSourceCannotBeCreated() { + final FlowDataSourceProvider provider = new FlowDataSourceProvider(); + provider.setUrl("jdbc:postgresql://localhost:5432/irrelevant"); // dedicated mode + provider.setDriverClass("com.example.NoSuchDriver"); // fails to load -> pool build throws + provider.init(); // must NOT throw + assertNull("no DataSource must be produced when creation fails", provider.getDataSource()); + provider.close(); // must be safe when nothing usable was created + } + + /** An injected provider that produced no DataSource — the exact blueprint scenario that used to fail. */ + private static FlowDataSourceProvider inertProvider() { + final FlowDataSourceProvider provider = new FlowDataSourceProvider(); + provider.setUrl("jdbc:postgresql://localhost:5432/irrelevant"); + provider.setDriverClass("com.example.NoSuchDriver"); + provider.init(); + assertNull(provider.getDataSource()); + return provider; + } + + @Test + public void repositoryIsInertWhenTheProviderHasNoDataSource() throws Exception { + final PostgresFlowRepository repository = new PostgresFlowRepository(new MetricRegistry()); + repository.setRunSchemaChangelog(false); + repository.setDataSourceProvider(inertProvider()); + repository.start(); // must NOT throw (feature loads) + repository.persist(Collections.singletonList(mock(Flow.class))); // must be a no-op, not an NPE + repository.stop(); + } + + @Test + public void queryServiceIsInertWhenTheProviderHasNoDataSource() throws Exception { + final PostgresFlowQueryService queryService = new PostgresFlowQueryService(); + queryService.setDataSourceProvider(inertProvider()); + queryService.start(); // must NOT throw (feature loads) + final List filters = Collections.singletonList(new TimeRangeFilter(0, 1000)); + final CompletableFuture result = queryService.getFlowCount(filters); + assertTrue("queries must fail cleanly, not NPE, when unconfigured", result.isCompletedExceptionally()); + queryService.stop(); + } +} From f6bb3a790a5fcaefe84dc66a0919f7784519c2b1 Mon Sep 17 00:00:00 2001 From: Dino Date: Thu, 9 Jul 2026 21:18:34 -0500 Subject: [PATCH 06/15] Add docs --- docs/modules/operation/nav.adoc | 1 + .../pages/deep-dive/flows/aggregation.adoc | 2 +- .../pages/deep-dive/flows/basic.adoc | 174 +++++++++--------- .../pages/deep-dive/flows/elasticsearch.adoc | 137 ++++++++++++++ .../deep-dive/flows/sentinel/sentinel.adoc | 63 ++++++- 5 files changed, 290 insertions(+), 87 deletions(-) create mode 100644 docs/modules/operation/pages/deep-dive/flows/elasticsearch.adoc diff --git a/docs/modules/operation/nav.adoc b/docs/modules/operation/nav.adoc index 2c6085634b10..ab536fda4a81 100644 --- a/docs/modules/operation/nav.adoc +++ b/docs/modules/operation/nav.adoc @@ -189,6 +189,7 @@ ** xref:deep-dive/flows/introduction.adoc[] *** xref:deep-dive/flows/basic.adoc[] +*** xref:deep-dive/flows/elasticsearch.adoc[] *** xref:deep-dive/flows/distributed.adoc[] *** xref:deep-dive/flows/sentinel/sentinel.adoc[] *** xref:deep-dive/flows/classification-engine.adoc[] diff --git a/docs/modules/operation/pages/deep-dive/flows/aggregation.adoc b/docs/modules/operation/pages/deep-dive/flows/aggregation.adoc index d2e823a2bd9e..5705ada5231b 100644 --- a/docs/modules/operation/pages/deep-dive/flows/aggregation.adoc +++ b/docs/modules/operation/pages/deep-dive/flows/aggregation.adoc @@ -4,7 +4,7 @@ The flow query engine supports rendering the top _N_ metrics from pre-aggregated documents stored in Elasticsearch. You can use these statistics to help alleviate computation load on the Elasticsearch cluster, particularly in environments with large volumes of flows (more than 10,000 per second). -To use this functionality, you must <> and set up the streaming analytics tool to process flows and persist aggregates in Elasticsearch. +To use this functionality, you must <> and set up the streaming analytics tool to process flows and persist aggregates in Elasticsearch. Set the following properties in `$\{OPENNMS_HOME}/etc/org.opennms.features.flows.persistence.elastic.cfg` to control the query engine to use aggregated flows: diff --git a/docs/modules/operation/pages/deep-dive/flows/basic.adoc b/docs/modules/operation/pages/deep-dive/flows/basic.adoc index 80178a8bd4f5..7b93355b3c59 100644 --- a/docs/modules/operation/pages/deep-dive/flows/basic.adoc +++ b/docs/modules/operation/pages/deep-dive/flows/basic.adoc @@ -1,36 +1,26 @@ [[flows-basic]] = Basic Flows Setup -:description: Get started with flows data collection in {page-component-title} including configuring Elasticsearch, Kafka, and protocol listeners. +:description: Get started with flows in {page-component-title} using the built-in PostgreSQL flow repository, stored in the OpenNMS database or a dedicated external database. -This section describes how to get started with flows to collect, enrich (classify), persist, and visualize flows. +This section describes how to get started with flows: collect, enrich (classify), persist, and visualize network flows using the built-in PostgreSQL flow repository. +With this option, {page-component-title} persists flows to a relational database -- by default the same PostgreSQL database {page-component-title} already uses -- so you can start collecting flows without deploying any additional storage. + +TIP: For high flow volumes, or to keep flow storage separate from the {page-component-title} database, persist flows to a dedicated external PostgreSQL database (see <>), or scale out with xref:deep-dive/flows/elasticsearch.adoc[Elasticsearch] and xref:deep-dive/flows/sentinel/sentinel.adoc[Sentinel]. == Requirements -Make sure that you have the following before you set up flows: +Make sure you have the following before you set up flows: -* A configured {page-component-title} instance. +* A configured {page-component-title} instance and its PostgreSQL database. * One or more devices that send flows visible to {page-component-title}, and that are monitored with SNMP. -* Elasticsearch cluster set up with the https://github.com/OpenNMS/elasticsearch-drift-plugin[Elasticsearch Drift plugin] installed on every Elasticsearch node. -** The Drift plugin persists and queries flows that {page-component-title} collects. -The Drift version must match the targeted Elasticsearch version. -** (Optional) Configure Elasticsearch variables like `search.max_buckets` or maximum heap size `ES-JAVA_OPTS` if the default values are not sufficient for your volume of flows or number of nodes. -** A retention process to remove old flow indices so the disk does not fill up (for example, "keep _X_ days of flows"). -See <>. -** {page-component-title} set up to monitor the Elasticsearch stack and generate an alarm if it goes down. * The OpenNMS plugin for Grafana configured to visualize flows. ** Configure the flow and performance data sources. -[WARNING] -==== -Flows are high volume, and {page-component-title} does not expire them. -Configure an index retention process before you collect flows in production, or the Elasticsearch disk can fill up. -See <>. -==== - -== Configure {page-component-title} to communicate with Elasticsearch +== Enable the PostgreSQL flow repository -{page-component-title} must be set up to communicate with Elasticsearch and persist the collected flows data in a defined directory: +The PostgreSQL flow repository is provided by the optional `opennms-flows-postgres` feature. +Installing it registers PostgreSQL as the flow store; when present, it takes precedence over the default Elasticsearch flow persistence, so flows are both written to and read from PostgreSQL. . Connect to your {page-component-title} xref:reference:karaf-shell/karaf-shell.adoc[Karaf shell]: + @@ -39,59 +29,108 @@ See <> or to xref:deep-dive/flows/elasticsearch.adoc[Elasticsearch] as your volume grows. + +=== Tune persistence settings (optional) + +You can tune the flow repository through the `org.opennms.features.flows.persistence.postgres` configuration: + +[source, karaf] ---- -elasticUrl = http://10.10.3.218:9200 <1> -connTimeout = 30000 -readTimeout = 300000 -settings.index.number_of_replicas = 0 -settings.index.number_of_shards=1 -settings.index.refresh_interval=10s -elasticIndexStrategy=daily +config:edit org.opennms.features.flows.persistence.postgres +config:property-set batchSize 1000 <1> +config:property-set flushIntervalMs 500 <2> +config:property-set queueCapacity 100000 <3> +config:property-set queryThreads 4 <4> +config:update ---- -<1> Replace with comma-separated list of Elasticsearch nodes. +<1> Maximum number of flows written per batch insert. +<2> Maximum time, in milliseconds, before a partial batch is flushed. +<3> Bounded write-queue capacity; when full, the newest flows are dropped (and counted) rather than blocking ingest. +<4> Number of threads used to service flow queries. -** See <> for a complete set of configuration options. +[[flows-postgres-external]] +== Use an external PostgreSQL database -== Configuring NetFlow Composable Templates +For high flow volumes, or to isolate flow storage from the {page-component-title} database, persist flows to a dedicated external PostgreSQL database. +This is also the model used when persisting flows from xref:deep-dive/flows/sentinel/sentinel.adoc[Sentinel] in a distributed deployment, where every Sentinel writer and the {page-component-title} core reader share the same flow database. -OpenNMS supports (https://www.elastic.co/docs/manage-data/data-store/templates)[composable templates] for more flexible and modular management of Elasticsearch index settings. -To enable composable templates for NetFlow data, update below configuration in Karaf: +. Set up the external database and its partition management (see <>) before pointing {page-component-title} at it. +. Point {page-component-title} at the external database: ++ [source, karaf] ---- -config:edit org.opennms.features.flows.persistence.elastic -config:property-set useComposableTemplates true +config:edit org.opennms.features.flows.persistence.postgres +config:property-set datasource.url jdbc:postgresql://db-host:5432/flowdb <1> +config:property-set datasource.username flow_user <2> +config:property-set datasource.password flow_password <3> +config:property-set datasource.databaseName flowdb <4> config:update ---- +<1> JDBC URL of the external flow database. When set, {page-component-title} builds a dedicated connection pool instead of using the internal database. +<2> Database user with read/write access to the flow database. +<3> Password for the database user. +<4> Name of the external flow database. -=== Managing NetFlow Templates and Policies +NOTE: When `datasource.url` is left blank (the default), {page-component-title} uses its internal database. +Setting it switches to the dedicated external database. -NetFlow component and index templates can be managed from the following directory: +[[flows-postgres-partman]] +=== Partition management on the external database -`$\{OPENNMS_HOME}/etc/netflow-templates` +The automatic partition maintenance built into {page-component-title} applies only to the internal database. +For an external flow database, manage partition creation and expiration with https://github.com/pgpartman/pg_partman[pg_partman] and https://github.com/citusdata/pg_cron[pg_cron]. -You can customize existing templates, add new ones, or define Index Lifecycle Management (ILM) policies within this folder. Any valid Elasticsearch template or ILM JSON placed here will be picked up and applied by the system. +A ready-to-use setup script, `flow-partman-setup.sql`, is provided with the `org.opennms.features.flows.postgres` source (under `contrib/`). +Run it once against the fresh external database to install the extensions, create the partitioned `flow` table, and schedule partition rotation, creation, and expiration: -When placing files in the netflow-templates directory: +[source, console] +---- +psql -h db-host -U flow_admin -d flowdb -f flow-partman-setup.sql +---- -* Files intended as ILM policies should have names that include keywords such as ilm, lifecycle, or policy. -* Files with names containing setting(s), mapping(s), alias(es), or component will be treated as component templates. -* Files with names containing index or composable will be recognized as index templates. +You can override the partition width, retention window, number of pre-created partitions, and maintenance schedule without editing the script: + +[source, console] +---- +psql -h db-host -U flow_admin -d flowdb \ + -v partition_interval="'1 hour'" \ + -v retention="'7 days'" \ + -v premake=6 \ + -v maintenance_schedule="'*/15 * * * *'" \ + -f flow-partman-setup.sql +---- +Because the script owns the schema, set `runSchemaChangelog=false` on every {page-component-title} node that connects to this database so the built-in schema migration does not conflict with pg_partman: +[source, karaf] +---- +config:edit org.opennms.features.flows.persistence.postgres +config:property-set runSchemaChangelog false +config:update +---- +IMPORTANT: The script requires the `pg_cron` and `pg_partman` (5.0 or later) extensions on the database server, and `pg_cron` must be configured to run in the flow database (`cron.database_name`). +The script header documents these prerequisites and all configurable settings. == Multi-protocol listener @@ -133,7 +172,7 @@ To enable one of these protocols, find the correct example `listener` and `adapt [source, console] ${OPENNMS_HOME}/bin/send-event.pl -p 'daemonName Telemetryd' uei.opennms.org/internal/reloadDaemonConfig -This configuration opens a UDP socket bound to `0.0.0.0:8877` to listen and process NetFlow v5 messages that are are forwarded to this port. +This configuration opens a UDP socket bound to `0.0.0.0:8877` to listen and process NetFlow v5 messages that are forwarded to this port. == Enable flows on your devices @@ -164,40 +203,6 @@ It supports the `$nodeId`, `$ifIndex`, `$start`, and `$end` placeholders. After the plugin is configured, an icon is displayed at the top-right corner of an SNMP resource graph indicating that flow data is available for the interface. If you have trouble during or after configuration, refer to xref:deep-dive/flows/troubleshooting.adoc[]. -[[kafka-forwarder-config]] -== Configure Kafka forwarder - -Flows enriched with {page-component-title} node data can be forwarded to Kafka and persisted. -By default, enriched flows are stored in the `flowsDocument` topic and the payloads are encoded using https://developers.google.com/protocol-buffers/[Google Protocol Buffers]. -See `flowdocument.proto` in the corresponding source definition for the model definitions. - -To enable JSON support, set `useJson` to `true`. - -Follow these steps to configure forwarding flows to Kafka: - -. Enable Kafka forwarding: -+ -[source, console] ----- -$ ssh -p 8101 admin@localhost -... -admin@opennms()> config:edit org.opennms.features.flows.persistence.elastic -admin@opennms()> config:property-set enableForwarding true -admin@opennms()> config:update ----- - -. Configure the Kafka server for flows: -+ -[source, console] ----- -$ ssh -p 8101 admin@localhost -... -admin@opennms()> config:edit org.opennms.features.flows.persistence.kafka -admin@opennms()> config:property-set bootstrap.servers 127.0.0.1:9092 -admin@opennms()> config:property-set topic opennms-flows -admin@opennms()> config:update ----- - == Next steps After you set up basic flows monitoring, you may want to do some of the following tasks: @@ -207,5 +212,6 @@ After you set up basic flows monitoring, you may want to do some of the followin You can create rules to override the default classifications (see xref:deep-dive/flows/classification-engine.adoc[]). * xref:deep-dive/flows/distributed.adoc[Enable remote flows data collection] with Minions. -* xref:deep-dive/flows/sentinel/sentinel.adoc[Scale to manage large volumes of flows data] with Sentinels. +* xref:deep-dive/flows/elasticsearch.adoc[Scale flow storage with Elasticsearch] for high volumes. +* xref:deep-dive/flows/sentinel/sentinel.adoc[Scale flow processing with Sentinel] to distribute the processing load. * Add https://github.com/OpenNMS/nephron[OpenNMS Nephron] for aggregation and streaming analytics. diff --git a/docs/modules/operation/pages/deep-dive/flows/elasticsearch.adoc b/docs/modules/operation/pages/deep-dive/flows/elasticsearch.adoc new file mode 100644 index 000000000000..379d6b16b469 --- /dev/null +++ b/docs/modules/operation/pages/deep-dive/flows/elasticsearch.adoc @@ -0,0 +1,137 @@ + +[[flows-elasticsearch]] += Scaling Flows with Elasticsearch +:description: Persist high volumes of flows in {page-component-title} to Elasticsearch, including the Drift plugin, composable templates, index retention, and the Kafka forwarder. + +For high flow volumes, or when you want a dedicated, horizontally scalable flows analytics store, persist flows to Elasticsearch instead of the {page-component-title} PostgreSQL database. +Elasticsearch is the long-standing {page-component-title} flow store and scales out across a cluster for large deployments. + +NOTE: This document covers only the Elasticsearch-specific persistence configuration. +Complete the xref:deep-dive/flows/basic.adoc#flows-basic[basic flow setup] first (protocol listeners, device configuration, and the Grafana data source), which applies regardless of the flow store. + +NOTE: If you enabled the `opennms-flows-postgres` feature during the xref:deep-dive/flows/basic.adoc[basic setup], it takes precedence over Elasticsearch. +To persist to and read from Elasticsearch instead, remove its `featuresBoot.d` file and uninstall the feature (`feature:uninstall opennms-flows-postgres`). + +== Requirements + +Make sure that you have the following before you set up flows persistence to Elasticsearch: + +* A xref:deep-dive/flows/basic.adoc#flows-basic[basic flows environment] set up. +* Elasticsearch cluster set up with the https://github.com/OpenNMS/elasticsearch-drift-plugin[Elasticsearch Drift plugin] installed on every Elasticsearch node. +** The Drift plugin persists and queries flows that {page-component-title} collects. +The Drift version must match the targeted Elasticsearch version. +** (Optional) Configure Elasticsearch variables like `search.max_buckets` or maximum heap size `ES-JAVA_OPTS` if the default values are not sufficient for your volume of flows or number of nodes. +** A retention process to remove old flow indices so the disk does not fill up (for example, "keep _X_ days of flows"). +See <>. +** {page-component-title} set up to monitor the Elasticsearch stack and generate an alarm if it goes down. +* The OpenNMS plugin for Grafana configured to visualize flows. +** Configure the flow and performance data sources. + +[WARNING] +==== +Flows are high volume, and {page-component-title} does not expire flows stored in Elasticsearch. +Configure an index retention process before you collect flows in production, or the Elasticsearch disk can fill up. +See <>. +==== + +== Configure {page-component-title} to communicate with Elasticsearch + +{page-component-title} must be set up to communicate with Elasticsearch and persist the collected flows data in a defined directory: + +. Connect to your {page-component-title} xref:reference:karaf-shell/karaf-shell.adoc[Karaf shell]: ++ +[source, console] +---- +ssh -p 8101 admin@localhost +---- + +. Edit `$\{OPENNMS_HOME}/etc/org.opennms.features.flows.persistence.elastic.cfg` to configure flow persistence to use your Elasticsearch cluster: ++ +[source, karaf] +---- +config:edit org.opennms.features.flows.persistence.elastic +config:property-set elasticUrl http://elastic:9200 +config:update +---- + +. (Optional) Edit or create `$\{OPENNMS_HOME}/etc/org.opennms.features.flows.persistence.elastic.cfg` and configure persistence settings: ++ +[source, xml] +---- +elasticUrl = http://10.10.3.218:9200 <1> +connTimeout = 30000 +readTimeout = 300000 +settings.index.number_of_replicas = 0 +settings.index.number_of_shards=1 +settings.index.refresh_interval=10s +elasticIndexStrategy=daily +---- +<1> Replace with comma-separated list of Elasticsearch nodes. + +** See <> for a complete set of configuration options. + +== Configuring NetFlow Composable Templates + +OpenNMS supports (https://www.elastic.co/docs/manage-data/data-store/templates)[composable templates] for more flexible and modular management of Elasticsearch index settings. +To enable composable templates for NetFlow data, update below configuration in Karaf: + +[source, karaf] +---- +config:edit org.opennms.features.flows.persistence.elastic +config:property-set useComposableTemplates true +config:update +---- + +=== Managing NetFlow Templates and Policies + +NetFlow component and index templates can be managed from the following directory: + +`$\{OPENNMS_HOME}/etc/netflow-templates` + +You can customize existing templates, add new ones, or define Index Lifecycle Management (ILM) policies within this folder. Any valid Elasticsearch template or ILM JSON placed here will be picked up and applied by the system. + +When placing files in the netflow-templates directory: + +* Files intended as ILM policies should have names that include keywords such as ilm, lifecycle, or policy. +* Files with names containing setting(s), mapping(s), alias(es), or component will be treated as component templates. +* Files with names containing index or composable will be recognized as index templates. + +[[kafka-forwarder-config]] +== Configure Kafka forwarder + +Flows enriched with {page-component-title} node data can be forwarded to Kafka and persisted. +By default, enriched flows are stored in the `flowsDocument` topic and the payloads are encoded using https://developers.google.com/protocol-buffers/[Google Protocol Buffers]. +See `flowdocument.proto` in the corresponding source definition for the model definitions. + +To enable JSON support, set `useJson` to `true`. + +Follow these steps to configure forwarding flows to Kafka: + +. Enable Kafka forwarding: ++ +[source, console] +---- +$ ssh -p 8101 admin@localhost +... +admin@opennms()> config:edit org.opennms.features.flows.persistence.elastic +admin@opennms()> config:property-set enableForwarding true +admin@opennms()> config:update +---- + +. Configure the Kafka server for flows: ++ +[source, console] +---- +$ ssh -p 8101 admin@localhost +... +admin@opennms()> config:edit org.opennms.features.flows.persistence.kafka +admin@opennms()> config:property-set bootstrap.servers 127.0.0.1:9092 +admin@opennms()> config:property-set topic opennms-flows +admin@opennms()> config:update +---- + +== Next steps + +* Classify data flows to resolve them to application names (see xref:deep-dive/flows/classification-engine.adoc[]). +* xref:deep-dive/flows/sentinel/sentinel.adoc[Scale flow processing with Sentinel] to distribute the processing load. +* Add https://github.com/OpenNMS/nephron[OpenNMS Nephron] for aggregation and streaming analytics. diff --git a/docs/modules/operation/pages/deep-dive/flows/sentinel/sentinel.adoc b/docs/modules/operation/pages/deep-dive/flows/sentinel/sentinel.adoc index 84a47c3eaac1..c872bf48a95b 100644 --- a/docs/modules/operation/pages/deep-dive/flows/sentinel/sentinel.adoc +++ b/docs/modules/operation/pages/deep-dive/flows/sentinel/sentinel.adoc @@ -1,13 +1,13 @@ [[flows-scaling]] = Scale Flow Processing with Sentinel -:description: Learn how to scale flows data collection in {page-component-title} with Sentinel; configure PosgreSQL database, Elasticsearch, and message brokers. +:description: Learn how to scale flows data collection in {page-component-title} with Sentinel; configure the PostgreSQL database, flow persistence to Elasticsearch or PostgreSQL, and message brokers. When your flows data collection volume increases to the point that {page-component-title} is too busy processing flows, you can add one or more Sentinels to do the processing instead. Sentinel can do the following: * Consume flow messages from Minions through a message broker (ActiveMQ or Apache Kafka) -* Persist network flow messages to Elasticsearch +* Persist network flow messages to Elasticsearch or PostgreSQL * Generate and send events to the {page-component-title} core instance via message broker .Flow integration with Sentinel @@ -30,6 +30,12 @@ Repeat the following steps on each Sentinel node. == Configure access to PostgreSQL database +Sentinel connects to the {page-component-title} core PostgreSQL database to enrich flows with node and interface metadata. +This is required regardless of where flows are persisted. + +NOTE: This is the {page-component-title} *core* database (used for enrichment), configured through the `org.opennms.netmgt.distributed.datasource` pid. +It is not the same as the flow database used when you <>. + .Connect to the Sentinel Karaf shell via SSH [source, console] ---- @@ -54,6 +60,9 @@ config:update == Configure access to Elasticsearch +Sentinel can persist flows either to Elasticsearch (this section) or to PostgreSQL (see <>). +Configure one flow-persistence backend. + .Connect to the Sentinel Karaf shell via SSH [source, console] ---- @@ -82,6 +91,56 @@ Additional configuration properties are documented in < +config:property-set datasource.username flow_user <2> +config:property-set datasource.password flow_password <3> +config:property-set datasource.databaseName flowdb <4> +config:property-set runSchemaChangelog false <5> +config:update +---- +<1> JDBC URL of the *dedicated flow database* -- not the {page-component-title} core database. +<2> Database user with read/write access to the flow database. +<3> Password for the database user. +<4> Name of the flow database. +<5> Let a single node own the schema (the {page-component-title} core reader, or one designated writer); set `false` on all other Sentinel writers. + +When installed, `sentinel-flows-postgres` takes precedence over Elasticsearch flow persistence on that Sentinel. +Kafka forwarding, if configured, continues to run alongside it. + +Set up partition management (creation, rotation, and expiration) on the external flow database with pg_partman and pg_cron using the `flow-partman-setup.sql` script. +See xref:operation:deep-dive/flows/basic.adoc#flows-postgres-partman[Partition management on the external database]. + +NOTE: For the {page-component-title} core instance to read these flows, install and configure the `opennms-flows-postgres` feature on the core, pointing it at the *same* flow database. +See xref:operation:deep-dive/flows/basic.adoc#flows-postgres-external[Use an external PostgreSQL database]. + == Set up message broker [{tabs}] From 2dbed89630f735df1bab68f639ea0a9595254240 Mon Sep 17 00:00:00 2001 From: Dino Date: Thu, 9 Jul 2026 21:19:59 -0500 Subject: [PATCH 07/15] Add health check for PostgreSQL flow on Sentinel --- .../src/main/resources/features-sentinel.xml | 2 + .../features/src/main/resources/features.xml | 2 + features/flows/postgres/pom.xml | 6 ++ .../postgres/PostgresFlowHealthCheck.java | 76 +++++++++++++++++++ .../OSGI-INF/blueprint/blueprint.xml | 8 ++ .../postgres/PostgresFlowDisabledTest.java | 13 ++++ .../postgres/PostgresFlowRepositoryIT.java | 19 +++++ 7 files changed, 126 insertions(+) create mode 100644 features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresFlowHealthCheck.java diff --git a/container/features/src/main/resources/features-sentinel.xml b/container/features/src/main/resources/features-sentinel.xml index a889d1532d10..77941a96e767 100644 --- a/container/features/src/main/resources/features-sentinel.xml +++ b/container/features/src/main/resources/features-sentinel.xml @@ -169,6 +169,8 @@ spring-jdbc opennms-core-db + + opennms-health-api mvn:com.fasterxml.jackson.core/jackson-databind/${jackson2Version} mvn:org.yaml/snakeyaml/1.31 mvn:org.liquibase/liquibase-core/${liquibaseVersion} diff --git a/container/features/src/main/resources/features.xml b/container/features/src/main/resources/features.xml index 689820955691..eb0d285bc927 100644 --- a/container/features/src/main/resources/features.xml +++ b/container/features/src/main/resources/features.xml @@ -1223,6 +1223,8 @@ opennms-core-db + + opennms-health-api mvn:com.fasterxml.jackson.core/jackson-databind/${jackson2Version} + + org.opennms.core.health + org.opennms.core.health.api + ${project.version} + org.opennms diff --git a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresFlowHealthCheck.java b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresFlowHealthCheck.java new file mode 100644 index 000000000000..f12aed2c30dc --- /dev/null +++ b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresFlowHealthCheck.java @@ -0,0 +1,76 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.postgres; + +import java.sql.Connection; +import java.util.List; +import java.util.Objects; + +import javax.sql.DataSource; + +import org.opennms.core.health.api.Context; +import org.opennms.core.health.api.HealthCheck; +import org.opennms.core.health.api.Response; +import org.opennms.core.health.api.Status; + +/** + * Health check for the PostgreSQL flow repository: verifies that a connection to the configured flow + * database (the {@code org.opennms.features.flows.persistence.postgres} pid) can be established and is + * valid. This is the PostgreSQL-store counterpart to the Elasticsearch "cluster health check for Flows", + * and is registered whenever the flows-postgres feature is installed. + */ +public class PostgresFlowHealthCheck implements HealthCheck { + + private static final int VALIDATION_TIMEOUT_SECONDS = 5; + + private final FlowDataSourceProvider dataSourceProvider; + + public PostgresFlowHealthCheck(final FlowDataSourceProvider dataSourceProvider) { + this.dataSourceProvider = Objects.requireNonNull(dataSourceProvider); + } + + @Override + public String getDescription() { + return "PostgreSQL database health check for Flows"; + } + + @Override + public List getTags() { + return List.of("flows", "postgres"); + } + + @Override + public Response perform(final Context context) { + final DataSource dataSource = dataSourceProvider.getDataSource(); + if (dataSource == null) { + return new Response(Status.Success, "Not configured"); + } + try (Connection connection = dataSource.getConnection()) { + if (connection.isValid(VALIDATION_TIMEOUT_SECONDS)) { + return new Response(Status.Success); + } + return new Response(Status.Failure, "The flow database connection is not valid."); + } catch (final Exception e) { + return new Response(e); + } + } +} diff --git a/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml index 2bf5c034344d..30c90bc2745a 100644 --- a/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ b/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@ -98,4 +98,12 @@ + + + + + + + \ No newline at end of file diff --git a/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowDisabledTest.java b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowDisabledTest.java index 63042cf43ec8..cd66649888d2 100644 --- a/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowDisabledTest.java +++ b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowDisabledTest.java @@ -21,6 +21,7 @@ */ package org.opennms.netmgt.flows.postgres; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; @@ -30,6 +31,8 @@ import java.util.concurrent.CompletableFuture; import org.junit.Test; +import org.opennms.core.health.api.Response; +import org.opennms.core.health.api.Status; import org.opennms.integration.api.v1.flows.Flow; import org.opennms.netmgt.flows.filter.api.Filter; import org.opennms.netmgt.flows.filter.api.TimeRangeFilter; @@ -84,4 +87,14 @@ public void queryServiceIsInertWhenTheProviderHasNoDataSource() throws Exception assertTrue("queries must fail cleanly, not NPE, when unconfigured", result.isCompletedExceptionally()); queryService.stop(); } + + @Test + public void healthCheckReportsNotConfiguredWhenNoDataSource() { + // Mirrors the Elasticsearch flow health check: an installed-but-unconfigured backend is + // reported as Success ("not configured"), not a Failure, so the check does not always fail. + final Response response = new PostgresFlowHealthCheck(inertProvider()).perform(null); + assertEquals(Status.Success, response.getStatus()); + assertTrue("message should indicate it is not configured", + response.getMessage() != null && response.getMessage().toLowerCase().contains("not configured")); + } } diff --git a/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepositoryIT.java b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepositoryIT.java index 5e49e86f23af..ecf9b2d96ebf 100644 --- a/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepositoryIT.java +++ b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepositoryIT.java @@ -299,6 +299,25 @@ public void dedicatedPoolProviderConnectsAndQueries() throws Exception { } } + @Test + public void healthCheckSucceedsAgainstTheDatabase() throws Exception { + final FlowDataSourceProvider provider = new FlowDataSourceProvider(); + provider.setUrl(JDBC_URL); + provider.setUsername(JDBC_USER); + provider.setPassword(JDBC_PASSWORD); + provider.setMinPool(1); + provider.setMaxPool(2); + provider.setMaxSize(2); + provider.init(); + try { + final org.opennms.core.health.api.Response response = + new PostgresFlowHealthCheck(provider).perform(null); + Assert.assertEquals(org.opennms.core.health.api.Status.Success, response.getStatus()); + } finally { + provider.close(); + } + } + private static Flow mockFlow(String application, Flow.Direction direction, long delta, long last, long bytes, String src, String dst) { Flow flow = Mockito.mock(Flow.class); Mockito.lenient().when(flow.getTimestamp()).thenReturn(Instant.ofEpochMilli(last)); From 1c44956e19e1f963ccc3366e76d23b0a2c230d28 Mon Sep 17 00:00:00 2001 From: Dino Date: Thu, 9 Jul 2026 21:22:55 -0500 Subject: [PATCH 08/15] Fix for NoClassDefFoundError on Sentinel die to missing opennms.home --- .../src/main/filtered-resources/etc/system.properties | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/features/container/sentinel/src/main/filtered-resources/etc/system.properties b/features/container/sentinel/src/main/filtered-resources/etc/system.properties index 3ae31b7fee34..ef3328d3eedd 100644 --- a/features/container/sentinel/src/main/filtered-resources/etc/system.properties +++ b/features/container/sentinel/src/main/filtered-resources/etc/system.properties @@ -39,6 +39,15 @@ org.ops4j.pax.logging.DefaultServiceLog.level = ERROR # OPENNMS: Change name to 'sentinel' karaf.name = sentinel +# +# OPENNMS: Several components (for example, the telemetry configuration model's PackageConfig) +# derive default paths from the opennms.home system property at class-load time. Set it to the +# Karaf home -- as the core container does -- so those static initializers do not fail with a +# NoClassDefFoundError when Sentinel processes telemetry/flows. Karaf resolves ${karaf.home} at +# runtime; the Docker entrypoint also sets -Dopennms.home, and an explicit -D still takes precedence. +# +opennms.home = ${karaf.home} + # # Default repository where bundles will be loaded from before using # other Maven repositories. For the full Maven configuration, see From fe7b7b9122a9eb55a616772408cf01c20a2b795f Mon Sep 17 00:00:00 2001 From: Dino Date: Fri, 10 Jul 2026 08:15:24 -0500 Subject: [PATCH 09/15] Clarify that /rest/flows/count defaults to last 4 hours --- docs/modules/development/pages/rest/flows.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/development/pages/rest/flows.adoc b/docs/modules/development/pages/rest/flows.adoc index e6c83cac373a..6d0e559a2fe4 100644 --- a/docs/modules/development/pages/rest/flows.adoc +++ b/docs/modules/development/pages/rest/flows.adoc @@ -14,7 +14,7 @@ NOTE: Unless otherwise specified, all units of time are expressed in millisecond | Resource | Description | /flows/count -| Retrieve the number of flows available. +| Retrieve the number of flows available. With no query parameters, the default filter is the previous four hours. | /flows/exporters | Retrieve basic information for the exporter nodes that have flows available. From ae78483f143043cb85e339a51cd01f52783f0420 Mon Sep 17 00:00:00 2001 From: Dino Date: Fri, 10 Jul 2026 17:00:23 -0500 Subject: [PATCH 10/15] Add metrics --- .../flows/postgres/BatchingFlowWriter.java | 28 +++++++++++++++++-- .../OSGI-INF/blueprint/blueprint.xml | 20 +++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/BatchingFlowWriter.java b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/BatchingFlowWriter.java index 635357568042..5cf077568c52 100644 --- a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/BatchingFlowWriter.java +++ b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/BatchingFlowWriter.java @@ -31,6 +31,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.codahale.metrics.Histogram; import com.codahale.metrics.Meter; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; @@ -59,6 +60,11 @@ public class BatchingFlowWriter implements AutoCloseable { private final Meter dropped; private final Meter flushed; private final Timer flushTimer; + private final Histogram batchSizeHistogram; + private final Histogram flushIntervalHistogram; + + /** Wall-clock time of the previous flush, used to derive the inter-flush interval. */ + private long lastFlushMs = 0L; private volatile boolean running = false; private Thread worker; @@ -81,8 +87,14 @@ public BatchingFlowWriter(final String name, this.flush = flush; this.enqueued = metrics.meter(MetricRegistry.name(name, "enqueued")); this.dropped = metrics.meter(MetricRegistry.name(name, "dropped")); + // flushed: total rows persisted; its rate is the throughput to PostgreSQL (flows/second). this.flushed = metrics.meter(MetricRegistry.name(name, "flushed")); + // flush: duration of each flush (batch INSERT) operation. this.flushTimer = metrics.timer(MetricRegistry.name(name, "flush")); + // batchSize: distribution of the number of rows written per flush. + this.batchSizeHistogram = metrics.histogram(MetricRegistry.name(name, "batchSize")); + // flushIntervalMs: distribution of the wall-clock interval between consecutive flushes. + this.flushIntervalHistogram = metrics.histogram(MetricRegistry.name(name, "flushIntervalMs")); metrics.register(MetricRegistry.name(name, "queueSize"), (com.codahale.metrics.Gauge) queue::size); } @@ -138,16 +150,28 @@ private void drainLoop() { } private void doFlush(final List batch) { + final int size = batch.size(); + recordFlushInterval(); try (Timer.Context ignored = flushTimer.time()) { flush.accept(new ArrayList<>(batch)); - flushed.mark(batch.size()); + flushed.mark(size); + batchSizeHistogram.update(size); } catch (final Exception e) { - LOG.warn("Failed to flush a batch of {} flow rows; the batch is dropped.", batch.size(), e); + LOG.warn("Failed to flush a batch of {} flow rows; the batch is dropped.", size, e); } finally { batch.clear(); } } + /** Record the wall-clock interval since the previous flush. Called only from the single drain thread. */ + private void recordFlushInterval() { + final long now = System.currentTimeMillis(); + if (lastFlushMs != 0L) { + flushIntervalHistogram.update(now - lastFlushMs); + } + lastFlushMs = now; + } + @Override public synchronized void close() { running = false; diff --git a/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml index 30c90bc2745a..5b6d39ee60a6 100644 --- a/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ b/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@ -39,6 +39,26 @@ + + + + + + + + + + + + + + + @@ -88,6 +92,7 @@ + From 1f3d8bb51539e1b9e24cc258fe0cd2b1029de5b4 Mon Sep 17 00:00:00 2001 From: Dino Date: Fri, 17 Jul 2026 13:13:45 -0500 Subject: [PATCH 12/15] use reWriteBatchedInserts, document synchronous_commit --- .../operation/pages/deep-dive/flows/basic.adoc | 13 +++++++++++++ .../flows/postgres/FlowDataSourceProvider.java | 14 +++++++++++++- .../flows/postgres/PostgresFlowRepository.java | 14 ++++++++++---- .../flows/postgres/PostgresFlowRepositoryIT.java | 3 +++ 4 files changed, 39 insertions(+), 5 deletions(-) diff --git a/docs/modules/operation/pages/deep-dive/flows/basic.adoc b/docs/modules/operation/pages/deep-dive/flows/basic.adoc index 8eca28da7abd..1f97a4ab3a48 100644 --- a/docs/modules/operation/pages/deep-dive/flows/basic.adoc +++ b/docs/modules/operation/pages/deep-dive/flows/basic.adoc @@ -100,6 +100,19 @@ config:update NOTE: When `datasource.url` is left blank (the default), {page-component-title} uses its internal database. Setting it switches to the dedicated external database. +=== Tune the external database for write throughput (optional) + +At high flow rates, two low-cost settings substantially raise write throughput on a dedicated flow database: + +* *Relax synchronous commit.* Flow ingest is high volume and loss-tolerant (the writer already drops flows under backpressure), so turning off synchronous commit removes the per-commit `fsync` wait for a large throughput gain. A crash may lose the last few hundred milliseconds of committed flows, but never risks corruption. Set it on the flow database (no restart required): ++ +[source, sql] +---- +ALTER DATABASE flowdb SET synchronous_commit = off; +---- + +* *Batched inserts are already optimized.* {page-component-title} appends `reWriteBatchedInserts=true` to the dedicated-pool JDBC URL, so batched flow inserts are sent to PostgreSQL as single multi-row statements. No action is required; do not remove the parameter if you set `datasource.url` with your own query string. + [[flows-postgres-partman]] === Partition management on the external database diff --git a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowDataSourceProvider.java b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowDataSourceProvider.java index 314f6378cd1e..611dc8ea0d9b 100644 --- a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowDataSourceProvider.java +++ b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowDataSourceProvider.java @@ -95,7 +95,7 @@ private DataSource createDataSource() throws Exception { cfg.setName("opennms-flows"); cfg.setDatabaseName(databaseName); cfg.setClassName(driverClass); - cfg.setUrl(url); + cfg.setUrl(withBatchInsertRewrite(url)); cfg.setUserName(username); cfg.setPassword(password); final HikariCPConnectionFactory pool = new HikariCPConnectionFactory(cfg); @@ -109,6 +109,18 @@ private DataSource createDataSource() throws Exception { return pool; } + /** + * Turn on the PostgreSQL driver's {@code reWriteBatchedInserts}, which collapses a batched INSERT into a + * single multi-row statement and substantially improves flow write throughput. Left untouched if the + * operator already set the parameter in datasource.url. + */ + private static String withBatchInsertRewrite(final String jdbcUrl) { + if (jdbcUrl.contains("reWriteBatchedInserts")) { + return jdbcUrl; + } + return jdbcUrl + (jdbcUrl.contains("?") ? '&' : '?') + "reWriteBatchedInserts=true"; + } + public DataSource getDataSource() { return dataSource; } diff --git a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepository.java b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepository.java index d0cb7471f24a..82c59c57466f 100644 --- a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepository.java +++ b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepository.java @@ -59,11 +59,14 @@ public class PostgresFlowRepository implements FlowRepository { private static final String CHANGELOG = "org/opennms/netmgt/flows/postgres/changelog.xml"; + // Plain placeholders (no ::inet/::jsonb casts) so the PostgreSQL driver's reWriteBatchedInserts can + // collapse the batch into a single multi-row INSERT. inet/jsonb values are bound with Types.OTHER in + // flush(), letting the server infer the column type. private static final String INSERT_SQL = "INSERT INTO flow (flow_ts, delta_switched, last_switched, first_switched, bytes, packets, " + "sampling_interval, direction, application, convo_key, src_addr, dst_addr, protocol, dscp, " + "exporter_node_id, input_snmp, output_snmp, location, document) VALUES " + - "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::inet, ?::inet, ?, ?, ?, ?, ?, ?, ?::jsonb)"; + "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; private final MetricRegistry metrics; private final ObjectMapper objectMapper = new ObjectMapper(); @@ -142,15 +145,18 @@ public void setValues(final PreparedStatement ps, final int i) throws SQLExcepti ps.setString(c++, r.direction); ps.setString(c++, r.application); ps.setString(c++, r.convoKey); - ps.setString(c++, r.srcAddr); - ps.setString(c++, r.dstAddr); + // inet columns: bind as OTHER (no ::inet cast) so the server infers the type and the batch + // stays rewritable; a null address becomes a SQL NULL. + ps.setObject(c++, r.srcAddr, Types.OTHER); + ps.setObject(c++, r.dstAddr, Types.OTHER); setInt(ps, c++, r.protocol); setInt(ps, c++, r.dscp); setInt(ps, c++, r.exporterNodeId); setInt(ps, c++, r.inputSnmp); setInt(ps, c++, r.outputSnmp); ps.setString(c++, r.location); - ps.setString(c, r.documentJson); + // jsonb column: bind as OTHER (no ::jsonb cast); the server casts the JSON text to jsonb. + ps.setObject(c, r.documentJson, Types.OTHER); } @Override diff --git a/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepositoryIT.java b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepositoryIT.java index ecf9b2d96ebf..a1772b0d899c 100644 --- a/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepositoryIT.java +++ b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowRepositoryIT.java @@ -97,6 +97,9 @@ public static void startContainer() throws Exception { DATA_SOURCE.setUrl(url); DATA_SOURCE.setUser(user); DATA_SOURCE.setPassword(password); + // Exercise the production write path: batched INSERTs are rewritten into a single multi-row + // statement, which requires the cast-free (plain-placeholder) INSERT + Types.OTHER binding. + DATA_SOURCE.setReWriteBatchedInserts(true); jdbc = new JdbcTemplate(DATA_SOURCE); repository = new PostgresFlowRepository(new MetricRegistry()); repository.setDataSource(DATA_SOURCE); From e98d8290378e4b165afecc3014dc1096bc500c4b Mon Sep 17 00:00:00 2001 From: Dino Date: Sun, 19 Jul 2026 20:40:47 -0500 Subject: [PATCH 13/15] Add retention info to docs; remove external retention section (script isn't commited anywhere yet); clarify purpose --- .../pages/deep-dive/flows/basic.adoc | 83 ++++++++++--------- 1 file changed, 43 insertions(+), 40 deletions(-) diff --git a/docs/modules/operation/pages/deep-dive/flows/basic.adoc b/docs/modules/operation/pages/deep-dive/flows/basic.adoc index 1f97a4ab3a48..addf990d927d 100644 --- a/docs/modules/operation/pages/deep-dive/flows/basic.adoc +++ b/docs/modules/operation/pages/deep-dive/flows/basic.adoc @@ -4,9 +4,9 @@ :description: Get started with flows in {page-component-title} using the built-in PostgreSQL flow repository, stored in the OpenNMS database or a dedicated external database. This section describes how to get started with flows: collect, enrich (classify), persist, and visualize network flows using the built-in PostgreSQL flow repository. -With this option, {page-component-title} persists flows to a relational database -- by default the same PostgreSQL database {page-component-title} already uses -- so you can start collecting flows without deploying any additional storage. +With this option, {page-component-title} persists flows to a relational database -- by default the same PostgreSQL database {page-component-title} already uses -- so you can start collecting flows without deploying any additional components. -TIP: For high flow volumes, or to keep flow storage separate from the {page-component-title} database, persist flows to a dedicated external PostgreSQL database (see <>), or scale out with xref:deep-dive/flows/elasticsearch.adoc[Elasticsearch] and xref:deep-dive/flows/sentinel/sentinel.adoc[Sentinel]. +TIP: Colocating flow storage in the `opennms` database is intended for low-volume or test configurations. For higher flow volumes, or to keep flow storage separate from the {page-component-title} database, persist flows to a dedicated external PostgreSQL database (see <>), or scale out with xref:deep-dive/flows/elasticsearch.adoc[Elasticsearch] and xref:deep-dive/flows/sentinel/sentinel.adoc[Sentinel]. == Requirements @@ -61,6 +61,7 @@ config:property-set flushIntervalMs 500 <2> config:property-set queueCapacity 100000 <3> config:property-set writerThreads 1 <4> config:property-set queryThreads 4 <5> +config:property-set runSchemaChangelog true <6> config:update ---- <1> Maximum number of flows written per batch insert. @@ -68,11 +69,49 @@ config:update <3> Bounded write-queue capacity; when full, the newest flows are dropped (and counted) rather than blocking ingest. <4> Number of concurrent writer threads. Each writes on its own database connection, so raising this parallelizes inserts across CPU cores on the database (a single writer thread can bottleneck on one database core at high volume). Defaults to 1. <5> Number of threads used to service flow queries. +<6> Execute the built-in schema changelog to create or update the `flow` table schema when `opennms-flows-postgres` is started. NOTE: Each writer and query thread uses one connection from the flow database connection pool while active. For a dedicated external database, keep `writerThreads` + `queryThreads` at or below the pool size (`connection.pool.maxPool`, default 25) and raise the pool if you increase them. When using the internal database, connections come from the shared {page-component-title} datasource. +=== Flow retention (optional) + +By default, {page-component-title} keeps *14 days* of flows in the internal database. +The `flow` table is range-partitioned by day, and a maintenance function pre-creates upcoming daily partitions and drops partitions older than the retention window. +Expiring old flows is therefore a fast partition drop rather than a row-by-row delete. + +The maintenance function takes the retention window and the number of days of partitions to pre-create, both in days: + +[source, sql] +---- +flow_maintain_partitions(retention_days, premake_days) +---- + +On the internal database it runs from the Vacuumd daemon with the defaults `flow_maintain_partitions(14, 3)`: keep 14 days of flows, and pre-create partitions 3 days ahead of ingest. + +To change the retention window, edit the flow-partition statement in `$\{OPENNMS_HOME}/etc/vacuumd-configuration.xml`: + +[source, xml] +---- + + DO $$ BEGIN IF EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'flow_maintain_partitions') + THEN PERFORM flow_maintain_partitions(30, 3); END IF; END $$; <1> + +---- +<1> Keep 30 days of flows instead of 14. Raise the second argument as well if flows arrive with timestamps more than a few days in the future. + +How often maintenance runs is set by the `period` attribute (in milliseconds) on the root `` element; the default of `86400000` runs it once every 24 hours. + +Then reload the Vacuumd configuration, or restart {page-component-title}: + +[source, console] +---- +${OPENNMS_HOME}/bin/send-event.pl uei.opennms.org/internal/reloadDaemonConfig --parm 'daemonName Vacuumd' +---- + +WARNING: Reducing the retention window drops the newly expired partitions on the next maintenance run, permanently deleting the flows they contain. + [[flows-postgres-external]] == Use an external PostgreSQL database @@ -90,12 +129,14 @@ config:property-set datasource.url jdbc:postgresql://db-host:5432/flowdb <1> config:property-set datasource.username flow_user <2> config:property-set datasource.password flow_password <3> config:property-set datasource.databaseName flowdb <4> +config:property-set runSchemaChangelog false <5> config:update ---- <1> JDBC URL of the external flow database. When set, {page-component-title} builds a dedicated connection pool instead of using the internal database. <2> Database user with read/write access to the flow database. <3> Password for the database user. <4> Name of the external flow database. +<5> Do not execute the built-in schema changelog to create or update the `flow` table schema when `opennms-flows-postgres` is started. NOTE: When `datasource.url` is left blank (the default), {page-component-title} uses its internal database. Setting it switches to the dedicated external database. @@ -113,44 +154,6 @@ ALTER DATABASE flowdb SET synchronous_commit = off; * *Batched inserts are already optimized.* {page-component-title} appends `reWriteBatchedInserts=true` to the dedicated-pool JDBC URL, so batched flow inserts are sent to PostgreSQL as single multi-row statements. No action is required; do not remove the parameter if you set `datasource.url` with your own query string. -[[flows-postgres-partman]] -=== Partition management on the external database - -The automatic partition maintenance built into {page-component-title} applies only to the internal database. -For an external flow database, manage partition creation and expiration with https://github.com/pgpartman/pg_partman[pg_partman] and https://github.com/citusdata/pg_cron[pg_cron]. - -A ready-to-use setup script, `flow-partman-setup.sql`, is provided with the `org.opennms.features.flows.postgres` source (under `contrib/`). -Run it once against the fresh external database to install the extensions, create the partitioned `flow` table, and schedule partition rotation, creation, and expiration: - -[source, console] ----- -psql -h db-host -U flow_admin -d flowdb -f flow-partman-setup.sql ----- - -You can override the partition width, retention window, number of pre-created partitions, and maintenance schedule without editing the script: - -[source, console] ----- -psql -h db-host -U flow_admin -d flowdb \ - -v partition_interval="'1 hour'" \ - -v retention="'7 days'" \ - -v premake=6 \ - -v maintenance_schedule="'*/15 * * * *'" \ - -f flow-partman-setup.sql ----- - -Because the script owns the schema, set `runSchemaChangelog=false` on every {page-component-title} node that connects to this database so the built-in schema migration does not conflict with pg_partman: - -[source, karaf] ----- -config:edit org.opennms.features.flows.persistence.postgres -config:property-set runSchemaChangelog false -config:update ----- - -IMPORTANT: The script requires the `pg_cron` and `pg_partman` (5.0 or later) extensions on the database server, and `pg_cron` must be configured to run in the flow database (`cron.database_name`). -The script header documents these prerequisites and all configurable settings. - == Multi-protocol listener With most tools, if you are monitoring multiple flow protocols, you must set up a listener on its own UDP port for each protocol. From 4fecbc4b6f8b658fb5decf59a80e1fe00d2f4e18 Mon Sep 17 00:00:00 2001 From: Dino Date: Fri, 24 Jul 2026 13:40:56 -0500 Subject: [PATCH 14/15] Roll in write-time aggregation, a la Nephron, into the PostgeSQL flow backend implementation --- .../src/main/resources/features-sentinel.xml | 5 +- .../pages/deep-dive/flows/basic.adoc | 166 +++++- .../deep-dive/flows/sentinel/sentinel.adoc | 36 +- features/flows/postgres/pom.xml | 7 + .../netmgt/flows/postgres/AggregatedFlow.java | 77 +++ .../postgres/AggregatedFlowQueryService.java | 508 ++++++++++++++++++ .../postgres/AggregatingFlowRepository.java | 148 +++++ .../postgres/FixedWindowAggregation.java | 132 +++++ .../netmgt/flows/postgres/FlowAggWriter.java | 101 ++++ .../netmgt/flows/postgres/FlowAggregator.java | 456 ++++++++++++++++ .../netmgt/flows/postgres/FlowInput.java | 109 ++++ .../postgres/PostgresSmartQueryService.java | 258 +++++++++ .../OSGI-INF/blueprint/blueprint.xml | 80 ++- .../netmgt/flows/postgres/changelog.xml | 163 +++++- .../postgres/FixedWindowAggregationTest.java | 222 ++++++++ .../netmgt/flows/postgres/FlowAggIT.java | 285 ++++++++++ .../flows/postgres/FlowAggregatorTest.java | 338 ++++++++++++ .../postgres/PostgresFlowDisabledTest.java | 40 ++ .../PostgresSmartQueryServiceTest.java | 89 +++ .../filtered/etc/vacuumd-configuration.xml | 24 +- 20 files changed, 3194 insertions(+), 50 deletions(-) create mode 100644 features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/AggregatedFlow.java create mode 100644 features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/AggregatedFlowQueryService.java create mode 100644 features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/AggregatingFlowRepository.java create mode 100644 features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FixedWindowAggregation.java create mode 100644 features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowAggWriter.java create mode 100644 features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowAggregator.java create mode 100644 features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowInput.java create mode 100644 features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresSmartQueryService.java create mode 100644 features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/FixedWindowAggregationTest.java create mode 100644 features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/FlowAggIT.java create mode 100644 features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/FlowAggregatorTest.java create mode 100644 features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresSmartQueryServiceTest.java diff --git a/container/features/src/main/resources/features-sentinel.xml b/container/features/src/main/resources/features-sentinel.xml index 77941a96e767..15e7f6fb46fb 100644 --- a/container/features/src/main/resources/features-sentinel.xml +++ b/container/features/src/main/resources/features-sentinel.xml @@ -163,8 +163,9 @@
Optional PostgreSQL-backed flow persistence for Sentinel (write path). When installed it registers a higher-ranked FlowRepository that overrides Elasticsearch persistence on this Sentinel; flow reads are - served by Horizon against the same database. Configure the flow database via the - org.opennms.features.flows.persistence.postgres pid (datasource.url/username/password/databaseName).
+ served by Horizon against the same database. Optional write-time aggregation into flow_agg can be enabled + per Sentinel (aggregation.enabled=true with a distinct aggregation.writerId). Configure the flow database + via the org.opennms.features.flows.persistence.postgres pid (datasource.url/username/password/databaseName). sentinel-flows spring-jdbc diff --git a/docs/modules/operation/pages/deep-dive/flows/basic.adoc b/docs/modules/operation/pages/deep-dive/flows/basic.adoc index addf990d927d..d2aac178ad98 100644 --- a/docs/modules/operation/pages/deep-dive/flows/basic.adoc +++ b/docs/modules/operation/pages/deep-dive/flows/basic.adoc @@ -44,7 +44,7 @@ echo "opennms-flows-postgres" | sudo tee ${OPENNMS_HOME}/etc/featuresBoot.d/flow ---- By default, no further configuration is required: flows are persisted to the {page-component-title} PostgreSQL database. -The flow table is partitioned by time and maintained automatically -- {page-component-title} pre-creates upcoming partitions and drops expired ones on a daily schedule. +The flow tables are partitioned by time and maintained automatically -- {page-component-title} pre-creates upcoming partitions and drops expired ones on a daily schedule (raw flows are kept 10 days by default; see <> to change this). NOTE: Storing flows in the {page-component-title} database is convenient for getting started and for small-to-moderate flow volumes. Because flows are high volume, monitor database growth and move to a <> or to xref:deep-dive/flows/elasticsearch.adoc[Elasticsearch] as your volume grows. @@ -75,31 +75,56 @@ NOTE: Each writer and query thread uses one connection from the flow database co For a dedicated external database, keep `writerThreads` + `queryThreads` at or below the pool size (`connection.pool.maxPool`, default 25) and raise the pool if you increase them. When using the internal database, connections come from the shared {page-component-title} datasource. +[[flows-postgres-retention]] === Flow retention (optional) -By default, {page-component-title} keeps *14 days* of flows in the internal database. -The `flow` table is range-partitioned by day, and a maintenance function pre-creates upcoming daily partitions and drops partitions older than the retention window. -Expiring old flows is therefore a fast partition drop rather than a row-by-row delete. +Both flow tables are range-partitioned by time, and a maintenance function pre-creates upcoming partitions and drops partitions older than the retention window. +Expiring old data is therefore a fast partition drop rather than a row-by-row delete. +Two tables are maintained independently, because they have very different write rates and value-over-time: -The maintenance function takes the retention window and the number of days of partitions to pre-create, both in days: +[cols="1,1,1,1", options="header"] +|=== +| Table | Contents | Default granularity | Default retention + +| `flow` +| Raw per-flow records (high volume, high write rate) +| hourly +| 10 days + +| `flow_agg` +| Write-time aggregates (sparse, low write rate; only present when <> is enabled) +| weekly +| 6 months +|=== + +The maintenance function takes the parent table, its partition-key column, a granularity, a retention interval, and the number of future buckets to pre-create: [source, sql] ---- -flow_maintain_partitions(retention_days, premake_days) +flow_maintain_partitions(parent, control_column, granularity, retention, premake) <1> ---- +<1> `granularity` is one of `hour`, `day`, `week`, or `month`. `retention` is a PostgreSQL `interval` (for example `'10 days'`, `'6 months'`). `premake` is a count of future buckets, so its wall-clock reach depends on the granularity -- 48 hourly buckets is two days ahead, 2 weekly buckets is two weeks ahead. -On the internal database it runs from the Vacuumd daemon with the defaults `flow_maintain_partitions(14, 3)`: keep 14 days of flows, and pre-create partitions 3 days ahead of ingest. +On the internal database it runs from the Vacuumd daemon with the defaults shown above. +Review these defaults *before deploying* the feature and adjust them to your data-volume and reporting needs -- the initial partition set is created from the same values when the schema is first installed. -To change the retention window, edit the flow-partition statement in `$\{OPENNMS_HOME}/etc/vacuumd-configuration.xml`: +To change retention or granularity, edit the flow-partition statement in `$\{OPENNMS_HOME}/etc/vacuumd-configuration.xml`: [source, xml] ---- - DO $$ BEGIN IF EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'flow_maintain_partitions') - THEN PERFORM flow_maintain_partitions(30, 3); END IF; END $$; <1> + DO $$ BEGIN + IF EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'flow_maintain_partitions') THEN + PERFORM flow_maintain_partitions('flow', 'flow_ts', 'hour', '30 days'::interval, 48); <1> + PERFORM flow_maintain_partitions('flow_agg', 'window_start', 'week', '1 year'::interval, 2); <2> + END IF; + END $$; ---- -<1> Keep 30 days of flows instead of 14. Raise the second argument as well if flows arrive with timestamps more than a few days in the future. +<1> Keep 30 days of raw flows instead of 10. Keep `premake` comfortably larger than a day's worth of buckets so daily maintenance never runs out between runs (48 hourly buckets = two days ahead). Switch to `'day'` granularity for lower-volume deployments where hourly partitions are unnecessarily fine. +<2> Keep one year of aggregates instead of six months. The `flow_agg` line is harmless when aggregation is not enabled -- the function is a no-op for a table that does not exist. + +NOTE: `premake` must stay ahead of the maintenance cadence. With the default once-per-day schedule, hourly partitions need well over 24 future buckets (the default is 48); weekly and monthly partitions only need one or two. How often maintenance runs is set by the `period` attribute (in milliseconds) on the root `` element; the default of `86400000` runs it once every 24 hours. @@ -110,7 +135,8 @@ Then reload the Vacuumd configuration, or restart {page-component-title}: ${OPENNMS_HOME}/bin/send-event.pl uei.opennms.org/internal/reloadDaemonConfig --parm 'daemonName Vacuumd' ---- -WARNING: Reducing the retention window drops the newly expired partitions on the next maintenance run, permanently deleting the flows they contain. +WARNING: Reducing the retention window drops the newly expired partitions on the next maintenance run, permanently deleting the data they contain. +Changing the *granularity* of a table that already holds data does not repartition existing partitions; the new granularity applies only to buckets created from then on, and older partitions age out normally under retention. [[flows-postgres-external]] == Use an external PostgreSQL database @@ -118,7 +144,7 @@ WARNING: Reducing the retention window drops the newly expired partitions on the For high flow volumes, or to isolate flow storage from the {page-component-title} database, persist flows to a dedicated external PostgreSQL database. This is also the model used when persisting flows from xref:deep-dive/flows/sentinel/sentinel.adoc[Sentinel] in a distributed deployment, where every Sentinel writer and the {page-component-title} core reader share the same flow database. -. Set up the external database and its partition management (see <>) before pointing {page-component-title} at it. +. Set up the external database -- create its schema and arrange for time-based partition maintenance -- before pointing {page-component-title} at it. . Point {page-component-title} at the external database: + @@ -154,6 +180,120 @@ ALTER DATABASE flowdb SET synchronous_commit = off; * *Batched inserts are already optimized.* {page-component-title} appends `reWriteBatchedInserts=true` to the dedicated-pool JDBC URL, so batched flow inserts are sent to PostgreSQL as single multi-row statements. No action is required; do not remove the parameter if you set `datasource.url` with your own query string. +[[flows-postgres-aggregation]] +== Reduce document volume with write-time aggregation (optional) + +Flows are high volume, so querying the raw `flow` table for every dashboard becomes expensive as volume grows. +Instead, {page-component-title} can aggregate flows as they arrive into fixed time windows and store compact per-dimension summaries in a `flow_agg` table; dashboards over longer time ranges then read those summaries rather than scanning raw flows. +This is the same model as flow aggregation with Nephron, but computed in-process by the `opennms-flows-postgres` feature -- no Apache Beam or Flink required -- and it runs on {page-component-title} core or on xref:deep-dive/flows/sentinel/sentinel.adoc[Sentinel]. + +=== What is aggregated + +For each fixed window (60 seconds by default), per exporter interface, the aggregator records: + +* the *interface total* (exact), and +* the *top-K* applications, conversations, and hosts by bytes, plus a single *Other* row that rolls up everything below the top-K. + +Each is kept both without TOS and per DSCP value, so traffic can be broken down by DSCP. +A flow's bytes are prorated across the windows it spans, using the same arithmetic as query-time proration, so the aggregated totals reconcile with the raw values. +The top-K cap bounds storage regardless of traffic shape: high-cardinality dimensions (conversations, hosts) collapse from potentially tens of thousands of rows per window to `top-K + 1` per exporter interface, while the interface total stays exact and the "Other" band captures the remaining traffic. + +=== Enable aggregation + +Aggregation is off by default because it adds write load alongside raw persistence. +Enable and tune it on the same configuration: + +[source, karaf] +---- +config:edit org.opennms.features.flows.persistence.postgres +config:property-set aggregation.enabled true <1> +config:property-set aggregation.windowSizeMs 60000 <2> +config:property-set aggregation.allowedLatenessMs 300000 <3> +config:property-set aggregation.topK 10 <4> +config:property-set aggregation.writerId core <5> +config:update +---- +<1> Turn on write-time aggregation. Flows continue to be stored raw as well. +<2> Fixed window size. This is the finest time resolution available to aggregated queries. +<3> How long a window stays open for late/out-of-order flows before it is flushed; flows arriving later than this for a closed window are dropped. +<4> Number of top applications/conversations/hosts kept per exporter interface per window; the remainder rolls into "Other". `0` keeps every key (no cap). +<5> Identifies this writer's rows. Give each Sentinel a distinct value; readers sum the partial rows across writers. + +NOTE: On the internal database, the `flow_agg` table is created automatically with the rest of the schema. On an external database (`runSchemaChangelog=false`), create `flow_agg` as part of your external schema setup, alongside the `flow` table. + +=== Serve dashboards from aggregates + +Reads go through a smart query router that chooses, per request, between two backends: + +* the *raw* `flow` table -- every flow, exact, and current to the last write, but expensive to scan over long ranges; and +* the *aggregated* `flow_agg` table -- compact per-window top-K summaries, cheap to read over long ranges, but top-K approximate and no fresher than the most recently closed window. + +The router mirrors the Elasticsearch `SmartQueryService`. It evaluates these rules in order and the first match wins: + +[cols="3,1,4", options="header"] +|=== +| Condition | Backend | Why + +| `smartQuery.alwaysUseRaw=true` (the default) +| Raw +| Aggregates are opt-in; reads are unchanged until you enable routing. + +| `smartQuery.alwaysUseAgg=true` +| Aggregated +| Forces aggregates for every eligible query (mainly for testing or benchmarking). + +| The query names specific entities +| Raw +| Aggregates keep only the top-K per window, so an arbitrarily named entity may have been rolled into "Other" and is not individually available; the raw table always has it. + +| No time-range filter +| Raw +| Aggregated reads need a bounded window to scope and sum; without one there is nothing to select. + +| Range duration >= `smartQuery.durationThresholdMs` (default 2 minutes) +| Aggregated +| Long ranges are exactly where scanning raw flows is expensive, and window-granularity results are acceptable there. + +| Range ends older than `smartQuery.endpointThresholdMs` (default 7 days) +| Aggregated +| Historical data is rarely needed at per-flow precision, the raw table may already be pruned by retention, and the far smaller aggregates can be kept longer. + +| Otherwise (a short, recent range) +| Raw +| Recent narrow ranges need exact, up-to-the-moment results; aggregates lag by up to one window plus `aggregation.allowedLatenessMs`, and scanning raw over a small range is cheap. +|=== + +"Specific-entity" queries are those that name which entities to return -- explicit application, conversation, or host lists, and the enumeration endpoints that autocomplete entity names in the UI. + +Even when a query is routed to the aggregated backend, only what `flow_agg` can answer is served from it; anything else transparently falls back to the raw table: + +[cols="3,2", options="header"] +|=== +| Query | Served from + +| Top-N application / conversation / host summaries and series | `flow_agg` +| DSCP field values, summaries, and series | `flow_agg` +| Flow count | Raw -- aggregates store no per-flow count, and counting per window would overcount flows that span windows +| Non-DSCP field queries | Raw -- only DSCP is aggregated +| Specific-entity summaries/series and entity enumeration | Raw -- always, per the routing rules above +|=== + +Aggregated reads are opt-in: + +[source, karaf] +---- +config:edit org.opennms.features.flows.persistence.postgres +config:property-set smartQuery.alwaysUseRaw false <1> +config:property-set smartQuery.durationThresholdMs 120000 <2> +config:property-set smartQuery.endpointThresholdMs 604800000 <3> +config:update +---- +<1> By default all reads use the raw table. Set this `false` for threshold-based routing, or set `smartQuery.alwaysUseAgg true` to force aggregates for eligible queries. +<2> Route to aggregates when the query range is at least this long (default 2 minutes). +<3> Route to aggregates when the query range ends further back in time than this (default 7 days). + +CAUTION: Aggregated results are top-K approximations at the window resolution. A per-entity byte total over a range can be slightly low for an entity that falls into "Other" in some windows (the interface total and the grand total remain exact), and a series `step` finer than `aggregation.windowSizeMs` cannot resolve below the window. Use short ranges, or `smartQuery.alwaysUseRaw true`, when you need exact per-flow precision. + == Multi-protocol listener With most tools, if you are monitoring multiple flow protocols, you must set up a listener on its own UDP port for each protocol. diff --git a/docs/modules/operation/pages/deep-dive/flows/sentinel/sentinel.adoc b/docs/modules/operation/pages/deep-dive/flows/sentinel/sentinel.adoc index c872bf48a95b..f254a82d0b0d 100644 --- a/docs/modules/operation/pages/deep-dive/flows/sentinel/sentinel.adoc +++ b/docs/modules/operation/pages/deep-dive/flows/sentinel/sentinel.adoc @@ -135,12 +135,44 @@ config:update When installed, `sentinel-flows-postgres` takes precedence over Elasticsearch flow persistence on that Sentinel. Kafka forwarding, if configured, continues to run alongside it. -Set up partition management (creation, rotation, and expiration) on the external flow database with pg_partman and pg_cron using the `flow-partman-setup.sql` script. -See xref:operation:deep-dive/flows/basic.adoc#flows-postgres-partman[Partition management on the external database]. +Because `runSchemaChangelog` is `false` on the Sentinel writers, create the flow schema and arrange for time-based partition maintenance on the external database out of band. A single node -- the {page-component-title} core reader or one designated writer -- may instead own schema creation with `runSchemaChangelog=true`. NOTE: For the {page-component-title} core instance to read these flows, install and configure the `opennms-flows-postgres` feature on the core, pointing it at the *same* flow database. See xref:operation:deep-dive/flows/basic.adoc#flows-postgres-external[Use an external PostgreSQL database]. +=== Reduce document volume with write-time aggregation (optional) + +Sentinel writers can also aggregate flows into the `flow_agg` table as they persist them, the same xref:operation:deep-dive/flows/basic.adoc#flows-postgres-aggregation[write-time aggregation] described for core. +Enable and tune it on the same configuration, and give each Sentinel its own `aggregation.writerId`: + +[source, karaf] +---- +config:edit org.opennms.features.flows.persistence.postgres +config:property-set aggregation.enabled true <1> +config:property-set aggregation.writerId sentinel-a <2> +config:update +---- +<1> Turn on write-time aggregation on this Sentinel; raw flows are still persisted. +<2> A stable id for this Sentinel. Each writer emits its own partial rows per window and the core reader sums the partials, so a distinct id per Sentinel keeps the origin of rows clear (summation is correct regardless of the id). + +Reads are always served by the {page-component-title} core reader -- Sentinel does not serve flow queries -- so configure the `smartQuery.*` routing there, not on the Sentinels. + +[[flows-sentinel-aggregation-affinity]] +==== Aggregation accuracy across multiple Sentinels + +Each Sentinel computes its top-K from only the flows it consumes, so correct top-K depends on all of a given exporter's flows reaching the *same* Sentinel. + +* With a *single Sentinel*, that node consumes the entire flow stream, so top-K is always exact. +* With *multiple Sentinels*, exact top-K depends on exporter affinity. By default it holds: the flow message broker keys each exporter's stream by `location@exporterIP:exporterPort` (the per-queue `use-routing-key` setting, `true` by default), so Kafka routes a given exporter consistently to one Sentinel. + +Regardless of distribution, *interface totals and the "Other" band are always exact* -- they are additive and reconcile across writers. +Only *top-K talker identification* can degrade to approximate (a large talker split across Sentinels may be undercounted while totals stay exact) when affinity is lost, which happens if: + +* a flow queue sets `use-routing-key="false"` in `telemetryd-configuration.xml` (sometimes done to spread a busy exporter's load across Sentinels), or +* an exporter sends from an unstable/rotating source UDP port, or is received at more than one location. + +If you cannot guarantee exporter affinity and need exact top-K, run aggregation on a single node that sees the whole flow stream (a single Sentinel, or one designated aggregating consumer) rather than on every Sentinel; otherwise the approximate top-K -- with exact totals -- is generally acceptable for dashboards. + == Set up message broker [{tabs}] diff --git a/features/flows/postgres/pom.xml b/features/flows/postgres/pom.xml index 67404865f4b3..38f42749cdc2 100644 --- a/features/flows/postgres/pom.xml +++ b/features/flows/postgres/pom.xml @@ -65,6 +65,13 @@ org.opennms.features.flows.processing ${project.version} + + + org.opennms.features.distributed + core-api + ${project.version} + provided + org.opennms.features.flows diff --git a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/AggregatedFlow.java b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/AggregatedFlow.java new file mode 100644 index 000000000000..2ba42076ea22 --- /dev/null +++ b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/AggregatedFlow.java @@ -0,0 +1,77 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.postgres; + +/** + * One aggregated row emitted when a window closes: the summed ingress/egress bytes for a single + * grouping key within a fixed time window. This is what {@link FlowAggregator} hands to its sink (for + * example {@link FlowAggWriter}) and what a reader sums back up per {@code (window, key)}. + */ +final class AggregatedFlow { + + /** The grouping dimension. TOS and top-K variants are added in a later phase. */ + enum Dimension { + /** Per-exporter-interface total (the uncapped parent used to reconstruct "Other"). */ + INTERFACE, + APPLICATION, + CONVERSATION, + HOST + } + + final long windowStartMs; + final long windowEndMs; + final int exporterNodeId; + final int ifIndex; + /** DSCP for a with-TOS aggregation; {@code null} for the without-TOS rollup (over all DSCP). */ + final Integer dscp; + final Dimension dimension; + /** The dimension value (application name, conversation key, host address); {@code null} for {@link Dimension#INTERFACE} and for "Other". */ + final String groupedByKey; + final long bytesIn; + final long bytesOut; + final boolean congestionEncountered; + final boolean nonEcnCapableTransport; + /** Resolved host name; only populated for {@link Dimension#HOST}. */ + final String hostname; + + AggregatedFlow(final long windowStartMs, final long windowEndMs, final int exporterNodeId, final int ifIndex, + final Integer dscp, final Dimension dimension, final String groupedByKey, final long bytesIn, + final long bytesOut, final boolean congestionEncountered, final boolean nonEcnCapableTransport, + final String hostname) { + this.windowStartMs = windowStartMs; + this.windowEndMs = windowEndMs; + this.exporterNodeId = exporterNodeId; + this.ifIndex = ifIndex; + this.dscp = dscp; + this.dimension = dimension; + this.groupedByKey = groupedByKey; + this.bytesIn = bytesIn; + this.bytesOut = bytesOut; + this.congestionEncountered = congestionEncountered; + this.nonEcnCapableTransport = nonEcnCapableTransport; + this.hostname = hostname; + } + + long bytesTotal() { + return bytesIn + bytesOut; + } +} diff --git a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/AggregatedFlowQueryService.java b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/AggregatedFlowQueryService.java new file mode 100644 index 000000000000..4ed9148b209c --- /dev/null +++ b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/AggregatedFlowQueryService.java @@ -0,0 +1,508 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.postgres; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.Function; +import java.util.function.Supplier; + +import javax.sql.DataSource; + +import org.opennms.netmgt.flows.api.Conversation; +import org.opennms.netmgt.flows.api.Directional; +import org.opennms.netmgt.flows.api.FlowQueryService; +import org.opennms.netmgt.flows.api.Host; +import org.opennms.netmgt.flows.api.LimitedCardinalityField; +import org.opennms.netmgt.flows.api.TrafficSummary; +import org.opennms.netmgt.flows.filter.api.DscpFilter; +import org.opennms.netmgt.flows.filter.api.ExporterNodeFilter; +import org.opennms.netmgt.flows.filter.api.Filter; +import org.opennms.netmgt.flows.filter.api.SnmpInterfaceIdFilter; +import org.opennms.netmgt.flows.filter.api.TimeRangeFilter; +import org.opennms.netmgt.flows.processing.ConversationKeyUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.jdbc.core.JdbcTemplate; + +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.Table; + +/** + * A {@link FlowQueryService} that answers queries from the pre-aggregated {@code flow_agg} table. + * + *

It is the counterpart to the Elasticsearch {@code AggregatedFlowQueryService}, and is meant to sit + * behind {@link PostgresSmartQueryService}, which only routes to it the queries aggregates can answer + * (non-specific-entity, time-bounded). It serves the topN summaries and topN series for + * application/conversation/host, plus DSCP field values/summaries/series (from the with-TOS + * INTERFACE rows), directly from {@code flow_agg}. Flow count, non-DSCP fields, entity listings, and + * explicit-set queries are delegated to the raw {@link FlowQueryService}. When no DataSource is + * configured or a query carries a filter the aggregates cannot honor, it also falls back to the delegate. + * + *

DSCP scope mirrors the ES TOS/non-TOS split: a {@link DscpFilter} selects the with-TOS rows + * ({@code dscp IN (...)}); its absence selects the without-TOS rollup ({@code dscp IS NULL}). "Other" is + * reconstructed from the dimension's own total (top-K rows + the stored null-key Other rows) minus the + * selected entries, so it stays exact per dimension. + */ +public class AggregatedFlowQueryService implements FlowQueryService { + + private static final Logger LOG = LoggerFactory.getLogger(AggregatedFlowQueryService.class); + private static final String OTHER = "Other"; + + private final FlowQueryService delegate; + private int threads = 4; + private FlowDataSourceProvider dataSourceProvider; + private DataSource dataSource; + private JdbcTemplate jdbcTemplate; + private ExecutorService executor; + + public AggregatedFlowQueryService(final FlowQueryService delegate) { + this.delegate = Objects.requireNonNull(delegate); + } + + public void start() { + if (this.dataSource == null && this.dataSourceProvider != null) { + this.dataSource = this.dataSourceProvider.getDataSource(); + } + if (this.dataSource == null) { + LOG.warn("AggregatedFlowQueryService has no flow DataSource; all queries will fall back to the raw service."); + return; + } + this.jdbcTemplate = new JdbcTemplate(this.dataSource); + this.executor = Executors.newFixedThreadPool(threads, r -> { + final Thread t = new Thread(r, "postgres-flow-agg-query"); + t.setDaemon(true); + return t; + }); + LOG.info("AggregatedFlowQueryService started (threads={}).", threads); + } + + public void stop() { + if (executor != null) { + executor.shutdownNow(); + } + } + + // ---- served from flow_agg (fall back to delegate when not servable) ---- + + @Override + public CompletableFuture>> getTopNApplicationSummaries(final int n, final boolean includeOther, final List filters) { + final AggWhere w = servable(filters); + return w == null ? delegate.getTopNApplicationSummaries(n, includeOther, filters) + : async(() -> summaries("APPLICATION", n, includeOther, w, s -> s)); + } + + @Override + public CompletableFuture>> getTopNConversationSummaries(final int n, final boolean includeOther, final List filters) { + final AggWhere w = servable(filters); + return w == null ? delegate.getTopNConversationSummaries(n, includeOther, filters) + : async(() -> summaries("CONVERSATION", n, includeOther, w, AggregatedFlowQueryService::toConversation)); + } + + @Override + public CompletableFuture>> getTopNHostSummaries(final int n, final boolean includeOther, final List filters) { + final AggWhere w = servable(filters); + return w == null ? delegate.getTopNHostSummaries(n, includeOther, filters) + : async(() -> summaries("HOST", n, includeOther, w, AggregatedFlowQueryService::toHost)); + } + + // ---- delegated (not yet served from aggregates, or inherently raw-only) ---- + + // Flow count is delegated to raw on purpose: flow_agg stores no per-flow count, and counting per + // window would overcount flows that span multiple windows. The raw COUNT(*) is exact and partition-pruned. + @Override + public CompletableFuture getFlowCount(final List filters) { + return delegate.getFlowCount(filters); + } + + @Override + public CompletableFuture> getApplications(final String matchingPrefix, final long limit, final List filters) { + return delegate.getApplications(matchingPrefix, limit, filters); + } + + @Override + public CompletableFuture>> getApplicationSummaries(final Set applications, final boolean includeOther, final List filters) { + return delegate.getApplicationSummaries(applications, includeOther, filters); + } + + @Override + public CompletableFuture, Long, Double>> getApplicationSeries(final Set applications, final long step, final boolean includeOther, final List filters) { + return delegate.getApplicationSeries(applications, step, includeOther, filters); + } + + @Override + public CompletableFuture, Long, Double>> getTopNApplicationSeries(final int n, final long step, final boolean includeOther, final List filters) { + final AggWhere w = servable(filters); + return w == null ? delegate.getTopNApplicationSeries(n, step, includeOther, filters) + : async(() -> series("APPLICATION", n, includeOther, step, w, s -> s)); + } + + @Override + public CompletableFuture> getConversations(final String locationPattern, final String protocolPattern, final String lowerIPPattern, final String upperIPPattern, final String applicationPattern, final long limit, final List filters) { + return delegate.getConversations(locationPattern, protocolPattern, lowerIPPattern, upperIPPattern, applicationPattern, limit, filters); + } + + @Override + public CompletableFuture>> getConversationSummaries(final Set conversations, final boolean includeOther, final List filters) { + return delegate.getConversationSummaries(conversations, includeOther, filters); + } + + @Override + public CompletableFuture, Long, Double>> getConversationSeries(final Set conversations, final long step, final boolean includeOther, final List filters) { + return delegate.getConversationSeries(conversations, step, includeOther, filters); + } + + @Override + public CompletableFuture, Long, Double>> getTopNConversationSeries(final int n, final long step, final boolean includeOther, final List filters) { + final AggWhere w = servable(filters); + return w == null ? delegate.getTopNConversationSeries(n, step, includeOther, filters) + : async(() -> series("CONVERSATION", n, includeOther, step, w, AggregatedFlowQueryService::toConversation)); + } + + @Override + public CompletableFuture> getHosts(final String regex, final long limit, final List filters) { + return delegate.getHosts(regex, limit, filters); + } + + @Override + public CompletableFuture>> getHostSummaries(final Set hosts, final boolean includeOther, final List filters) { + return delegate.getHostSummaries(hosts, includeOther, filters); + } + + @Override + public CompletableFuture, Long, Double>> getHostSeries(final Set hosts, final long step, final boolean includeOther, final List filters) { + return delegate.getHostSeries(hosts, step, includeOther, filters); + } + + @Override + public CompletableFuture, Long, Double>> getTopNHostSeries(final int n, final long step, final boolean includeOther, final List filters) { + final AggWhere w = servable(filters); + return w == null ? delegate.getTopNHostSeries(n, step, includeOther, filters) + : async(() -> series("HOST", n, includeOther, step, w, AggregatedFlowQueryService::toHost)); + } + + // Only DSCP is available from aggregates (the with-TOS INTERFACE rows); other fields delegate to raw. + @Override + public CompletableFuture> getFieldValues(final LimitedCardinalityField field, final List filters) { + final AggWhere w = servable(filters); + if (field != LimitedCardinalityField.DSCP || w == null) { + return delegate.getFieldValues(field, filters); + } + return async(() -> { + final List out = new ArrayList<>(); + jdbcTemplate.query("SELECT DISTINCT dscp FROM flow_agg WHERE dimension = 'INTERFACE'" + + scopeBase(w) + dscpFieldScope(w) + " ORDER BY dscp", + rs -> { out.add(rs.getString(1)); }); + return out; + }); + } + + @Override + public CompletableFuture>> getFieldSummaries(final LimitedCardinalityField field, final List filters) { + final AggWhere w = servable(filters); + if (field != LimitedCardinalityField.DSCP || w == null) { + return delegate.getFieldSummaries(field, filters); + } + return async(() -> { + final List> out = new ArrayList<>(); + jdbcTemplate.query("SELECT dscp::text AS entity, COALESCE(SUM(bytes_in),0) AS bin," + + " COALESCE(SUM(bytes_out),0) AS bout, bool_or(congestion_encountered) AS cong," + + " bool_or(non_ecn_capable_transport) AS nonect" + + " FROM flow_agg WHERE dimension = 'INTERFACE'" + scopeBase(w) + dscpFieldScope(w) + + " GROUP BY dscp ORDER BY (SUM(bytes_in)+SUM(bytes_out)) DESC", rs -> { + final InOut io = new InOut(); + io.in = rs.getLong("bin"); + io.out = rs.getLong("bout"); + io.cong = rs.getBoolean("cong"); + io.nonect = rs.getBoolean("nonect"); + out.add(summary(rs.getString("entity"), io)); + }); + return out; + }); + } + + @Override + public CompletableFuture, Long, Double>> getFieldSeries(final LimitedCardinalityField field, final long step, final List filters) { + final AggWhere w = servable(filters); + if (field != LimitedCardinalityField.DSCP || w == null) { + return delegate.getFieldSeries(field, step, filters); + } + return async(() -> { + final long s = step > 0 ? step : 1L; + final String bucket = "(((extract(epoch from window_start) * 1000)::bigint) / " + s + ") * " + s; + final Table, Long, Double> table = HashBasedTable.create(); + jdbcTemplate.query("SELECT dscp::text AS entity, " + bucket + " AS bucket_ms," + + " SUM(bytes_in) AS bin, SUM(bytes_out) AS bout" + + " FROM flow_agg WHERE dimension = 'INTERFACE'" + scopeBase(w) + dscpFieldScope(w) + + " GROUP BY entity, bucket_ms", rs -> { + final String e = rs.getString("entity"); + final long b = rs.getLong("bucket_ms"); + final long bin = rs.getLong("bin"); + final long bout = rs.getLong("bout"); + if (bin != 0) table.put(new Directional<>(e, true), b, (double) bin); + if (bout != 0) table.put(new Directional<>(e, false), b, (double) bout); + }); + return table; + }); + } + + // ---- aggregation query implementation ---- + + private CompletableFuture async(final Supplier supplier) { + return CompletableFuture.supplyAsync(supplier, executor); + } + + /** Parse into an aggregate WHERE, or {@code null} if the aggregates cannot honor this filter set. */ + private AggWhere servable(final List filters) { + if (executor == null) { + return null; // no DataSource -> not started -> fall back + } + final AggWhere w = new AggWhere(); + boolean haveTime = false; + for (final Filter filter : filters) { + if (filter instanceof TimeRangeFilter) { + final TimeRangeFilter t = (TimeRangeFilter) filter; + w.start = t.getStart(); + w.end = t.getEnd(); + haveTime = true; + } else if (filter instanceof SnmpInterfaceIdFilter) { + w.ifIndex = ((SnmpInterfaceIdFilter) filter).getSnmpInterfaceId(); + } else if (filter instanceof DscpFilter) { + w.dscp = ((DscpFilter) filter).getDscp(); + } else if (filter instanceof ExporterNodeFilter) { + final org.opennms.netmgt.flows.filter.api.NodeCriteria c = ((ExporterNodeFilter) filter).getCriteria(); + if (c.getNodeId() == null) { + return null; // flow_agg has no foreign-source/id columns; let the raw service handle it + } + w.exporterNodeId = c.getNodeId(); + } else { + return null; // unknown filter type -> not servable from aggregates + } + } + return haveTime ? w : null; + } + + /** Full scope for a per-entity dimension query: range + exporter/interface + TOS scope. */ + private String scopeSql(final AggWhere w) { + return scopeBase(w) + dscpScope(w); + } + + /** Range + optional exporter/interface, without any DSCP predicate. */ + private String scopeBase(final AggWhere w) { + final StringBuilder sb = new StringBuilder(); + sb.append(" AND window_start >= to_timestamp(").append(w.start).append("/1000.0)") + .append(" AND window_start < to_timestamp(").append(w.end).append("/1000.0)"); + if (w.exporterNodeId != null) { + sb.append(" AND exporter_node_id = ").append(w.exporterNodeId.intValue()); + } + if (w.ifIndex != null) { + sb.append(" AND if_index = ").append(w.ifIndex.intValue()); + } + return sb.toString(); + } + + private String dscpIn(final AggWhere w) { + final StringBuilder sb = new StringBuilder(" AND dscp IN ("); + for (int i = 0; i < w.dscp.size(); i++) { + if (i > 0) sb.append(','); + sb.append(w.dscp.get(i).intValue()); + } + return sb.append(')').toString(); + } + + /** TOS scope for dimension queries: an explicit DscpFilter selects with-TOS rows; else the rollup. */ + private String dscpScope(final AggWhere w) { + return (w.dscp != null && !w.dscp.isEmpty()) ? dscpIn(w) : " AND dscp IS NULL"; + } + + /** TOS scope for a per-DSCP field breakdown: all with-TOS rows (optionally restricted by DscpFilter). */ + private String dscpFieldScope(final AggWhere w) { + return (w.dscp != null && !w.dscp.isEmpty()) ? dscpIn(w) : " AND dscp IS NOT NULL"; + } + + private List> summaries(final String dimension, final int topN, final boolean includeOther, + final AggWhere w, final Function toEntity) { + final String scope = scopeSql(w); + final StringBuilder sql = new StringBuilder("SELECT grouped_by_key AS entity,") + .append(" COALESCE(SUM(bytes_in),0) AS bin, COALESCE(SUM(bytes_out),0) AS bout,") + .append(" bool_or(congestion_encountered) AS cong, bool_or(non_ecn_capable_transport) AS nonect") + .append(" FROM flow_agg WHERE dimension = '").append(dimension).append('\'').append(scope) + .append(" AND grouped_by_key IS NOT NULL") + .append(" GROUP BY grouped_by_key ORDER BY (SUM(bytes_in)+SUM(bytes_out)) DESC"); + if (topN > 0) { + sql.append(" LIMIT ").append(topN); + } + final Map selected = new LinkedHashMap<>(); + jdbcTemplate.query(sql.toString(), rs -> { + final InOut io = new InOut(); + io.in = rs.getLong("bin"); + io.out = rs.getLong("bout"); + io.cong = rs.getBoolean("cong"); + io.nonect = rs.getBoolean("nonect"); + selected.put(rs.getString("entity"), io); + }); + + final List> out = new ArrayList<>(); + long selIn = 0; + long selOut = 0; + for (final Map.Entry e : selected.entrySet()) { + out.add(summary(toEntity.apply(e.getKey()), e.getValue())); + selIn += e.getValue().in; + selOut += e.getValue().out; + } + if (includeOther) { + // Dimension total = every row of this dimension (top-K + the stored null-key Other rows). + final InOut total = dimensionTotal(dimension, scope); + final InOut other = new InOut(); + other.in = Math.max(0, total.in - selIn); + other.out = Math.max(0, total.out - selOut); + out.add(summary(toEntity.apply(OTHER), other)); + } + return out; + } + + /** + * Time series for the top-N entities of a dimension over the requested {@code step}, plus an "Other" + * series. Each flow_agg window is assigned to the step bucket containing its start (epoch-aligned, + * matching the raw service); the aggregation window size is the finest resolution available. + */ + private Table, Long, Double> series(final String dimension, final int topN, + final boolean includeOther, final long step, final AggWhere w, final Function toEntity) { + final long s = step > 0 ? step : 1L; + final String scope = scopeSql(w); + final String bucket = "(((extract(epoch from window_start) * 1000)::bigint) / " + s + ") * " + s; + + // 1. the top-N entities over the whole range + final List keys = new ArrayList<>(); + final StringBuilder keySql = new StringBuilder("SELECT grouped_by_key FROM flow_agg WHERE dimension = '") + .append(dimension).append('\'').append(scope) + .append(" AND grouped_by_key IS NOT NULL GROUP BY grouped_by_key") + .append(" ORDER BY (SUM(bytes_in)+SUM(bytes_out)) DESC"); + if (topN > 0) { + keySql.append(" LIMIT ").append(topN); + } + jdbcTemplate.query(keySql.toString(), rs -> { keys.add(rs.getString(1)); }); + + final Table, Long, Double> table = HashBasedTable.create(); + final Map selectedByBucket = new HashMap<>(); // bucket -> [in,out] over selected keys + + // 2. per selected entity, per bucket + if (!keys.isEmpty()) { + final StringBuilder in = new StringBuilder(); + final List params = new ArrayList<>(); + for (int i = 0; i < keys.size(); i++) { + if (i > 0) in.append(','); + in.append('?'); + params.add(keys.get(i)); + } + final String sql = "SELECT grouped_by_key AS entity, " + bucket + " AS bucket_ms," + + " SUM(bytes_in) AS bin, SUM(bytes_out) AS bout" + + " FROM flow_agg WHERE dimension = '" + dimension + "'" + scope + + " AND grouped_by_key IN (" + in + ")" + + " GROUP BY entity, bucket_ms"; + jdbcTemplate.query(sql, rs -> { + final T entity = toEntity.apply(rs.getString("entity")); + final long b = rs.getLong("bucket_ms"); + final long bin = rs.getLong("bin"); + final long bout = rs.getLong("bout"); + if (bin != 0) table.put(new Directional<>(entity, true), b, (double) bin); + if (bout != 0) table.put(new Directional<>(entity, false), b, (double) bout); + final long[] acc = selectedByBucket.computeIfAbsent(b, k -> new long[2]); + acc[0] += bin; + acc[1] += bout; + }, params.toArray()); + } + + // 3. Other per bucket = (dimension total in bucket) - (selected in bucket) + if (includeOther) { + final T other = toEntity.apply(OTHER); + final String totalSql = "SELECT " + bucket + " AS bucket_ms, SUM(bytes_in) AS bin, SUM(bytes_out) AS bout" + + " FROM flow_agg WHERE dimension = '" + dimension + "'" + scope + + " GROUP BY bucket_ms"; + jdbcTemplate.query(totalSql, rs -> { + final long b = rs.getLong("bucket_ms"); + final long[] sel = selectedByBucket.getOrDefault(b, new long[2]); + final long oin = Math.max(0, rs.getLong("bin") - sel[0]); + final long oout = Math.max(0, rs.getLong("bout") - sel[1]); + if (oin != 0) table.put(new Directional<>(other, true), b, (double) oin); + if (oout != 0) table.put(new Directional<>(other, false), b, (double) oout); + }); + } + return table; + } + + private InOut dimensionTotal(final String dimension, final String scope) { + final String sql = "SELECT COALESCE(SUM(bytes_in),0) AS bin, COALESCE(SUM(bytes_out),0) AS bout" + + " FROM flow_agg WHERE dimension = '" + dimension + "'" + scope; + final InOut io = new InOut(); + jdbcTemplate.query(sql, rs -> { + io.in = rs.getLong("bin"); + io.out = rs.getLong("bout"); + }); + return io; + } + + private static TrafficSummary summary(final T entity, final InOut io) { + return TrafficSummary.from(entity) + .withBytes(io.in, io.out) + .withCongestionEncountered(io.cong) + .withNonEcnCapableTransport(io.nonect) + .build(); + } + + private static Conversation toConversation(final String key) { + return OTHER.equals(key) ? Conversation.forOther().build() + : Conversation.from(ConversationKeyUtils.fromJsonString(key)).build(); + } + + private static Host toHost(final String ip) { + return OTHER.equals(ip) ? Host.forOther().build() : Host.from(ip).build(); + } + + private static final class AggWhere { + long start; + long end; + Integer exporterNodeId; + Integer ifIndex; + List dscp; // null/empty -> without-TOS scope + } + + private static final class InOut { + long in; + long out; + boolean cong; + boolean nonect; + } + + // --- config setters (blueprint) --- + public void setThreads(final int threads) { this.threads = threads; } + public void setDataSourceProvider(final FlowDataSourceProvider dataSourceProvider) { this.dataSourceProvider = dataSourceProvider; } + public void setDataSource(final DataSource dataSource) { this.dataSource = dataSource; } +} diff --git a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/AggregatingFlowRepository.java b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/AggregatingFlowRepository.java new file mode 100644 index 000000000000..21bf37ceb0d3 --- /dev/null +++ b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/AggregatingFlowRepository.java @@ -0,0 +1,148 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.postgres; + +import java.util.Collection; +import java.util.Objects; + +import javax.sql.DataSource; + +import org.opennms.distributed.core.api.Identity; +import org.opennms.integration.api.v1.flows.Flow; +import org.opennms.integration.api.v1.flows.FlowException; +import org.opennms.integration.api.v1.flows.FlowRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.codahale.metrics.MetricRegistry; + +/** + * A {@link FlowRepository} that feeds the write-time {@link FlowAggregator} instead of storing raw flows. + * Registered as a forwarder (see the blueprint {@code flows.repository.forwarder=true}) so the + * pipeline always delivers flows to it, alongside whichever primary store is active — it never competes + * with {@link PostgresFlowRepository} for the highest-ranked-store tier. + * + *

Lifecycle mirrors {@link PostgresFlowRepository}: the DataSource is resolved from the shared + * {@link FlowDataSourceProvider} in {@link #start()}, and the repository stays inert (a no-op + * {@link #persist}) rather than failing to load when aggregation is disabled or no DataSource is + * configured. On Horizon core this is the single aggregating writer; on Sentinel each instance sets its + * own {@code writerId} and emits partial rows that readers sum. + */ +public class AggregatingFlowRepository implements FlowRepository { + + private static final Logger LOG = LoggerFactory.getLogger(AggregatingFlowRepository.class); + + private final MetricRegistry metrics; + + private boolean enabled = false; + private long windowSizeMs = 60_000L; + private long allowedLatenessMs = 300_000L; + private long flushIntervalMs = 5_000L; + private long idleFlushMs = 0L; // 0 = auto (windowSize + lateness + 60s) + private int topK = 10; + private String writerId = ""; // blank -> auto-default to the node Identity (see resolveWriterId) + + private Identity identity; + private FlowDataSourceProvider dataSourceProvider; + private DataSource dataSource; + private FlowAggregator aggregator; + + public AggregatingFlowRepository(final MetricRegistry metrics) { + this.metrics = Objects.requireNonNull(metrics); + } + + public void start() { + if (!enabled) { + LOG.info("Write-time flow aggregation is disabled (set aggregation.enabled=true on the " + + "org.opennms.features.flows.persistence.postgres pid to enable it)."); + return; + } + if (this.dataSource == null && this.dataSourceProvider != null) { + this.dataSource = this.dataSourceProvider.getDataSource(); + } + if (this.dataSource == null) { + LOG.error("Write-time flow aggregation not started: no flow DataSource is configured. Flows will " + + "NOT be aggregated until datasource.url is set on the " + + "org.opennms.features.flows.persistence.postgres pid."); + return; + } + final String effectiveWriterId = resolveWriterId(); + final FlowAggWriter writer = new FlowAggWriter(this.dataSource, effectiveWriterId); + this.aggregator = new FlowAggregator(windowSizeMs, allowedLatenessMs, flushIntervalMs, topK, idleFlushMs, writer, metrics); + this.aggregator.start(); + LOG.info("Write-time flow aggregation started (writerId={}, windowSizeMs={}, allowedLatenessMs={}, " + + "flushIntervalMs={}, topK={}, idleFlushMs={}).", effectiveWriterId, windowSizeMs, allowedLatenessMs, + flushIntervalMs, topK, idleFlushMs); + } + + public void stop() { + if (aggregator != null) { + aggregator.close(); + } + LOG.info("Write-time flow aggregation stopped."); + } + + @Override + public void persist(final Collection flows) throws FlowException { + final FlowAggregator agg = this.aggregator; + if (agg == null || flows == null || flows.isEmpty()) { + // Disabled, inert (no DataSource), or nothing to do. + return; + } + for (final Flow flow : flows) { + agg.add(flow); + } + } + + /** + * The writer id to tag this instance's rows with: the explicitly configured {@code writerId} if set, + * otherwise the node's {@link Identity} id (so each Sentinel gets a distinct id automatically), falling + * back to {@code "core"} when no Identity is available. + */ + String resolveWriterId() { + if (writerId != null && !writerId.trim().isEmpty()) { + return writerId; + } + if (identity != null && identity.getId() != null && !identity.getId().trim().isEmpty()) { + return identity.getId(); + } + return "core"; + } + + // --- config setters (blueprint) --- + /** Node identity used to auto-default {@code writerId} when it is left blank. */ + public void setIdentity(final Identity identity) { this.identity = identity; } + /** Blueprint wiring: supplies the flow DataSource in start() (may be inert, i.e. return null). */ + public void setDataSourceProvider(final FlowDataSourceProvider dataSourceProvider) { this.dataSourceProvider = dataSourceProvider; } + /** Test/embedding hook: use this DataSource directly instead of resolving one from the provider. */ + public void setDataSource(final DataSource dataSource) { this.dataSource = dataSource; } + public void setEnabled(final boolean enabled) { this.enabled = enabled; } + public void setWindowSizeMs(final long windowSizeMs) { this.windowSizeMs = windowSizeMs; } + public void setAllowedLatenessMs(final long allowedLatenessMs) { this.allowedLatenessMs = allowedLatenessMs; } + public void setFlushIntervalMs(final long flushIntervalMs) { this.flushIntervalMs = flushIntervalMs; } + /** Wall-clock age after which an open window is flushed even if the watermark stalls; <= 0 auto-derives. */ + public void setIdleFlushMs(final long idleFlushMs) { this.idleFlushMs = idleFlushMs; } + /** Per (exporter, interface, dimension) cap for application/conversation/host; <= 0 disables capping. */ + public void setTopK(final int topK) { this.topK = topK; } + /** Identifies this writer's partial rows; give each Sentinel a distinct value so readers sum correctly. */ + public void setWriterId(final String writerId) { this.writerId = writerId; } +} diff --git a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FixedWindowAggregation.java b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FixedWindowAggregation.java new file mode 100644 index 000000000000..a54ecc004ddb --- /dev/null +++ b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FixedWindowAggregation.java @@ -0,0 +1,132 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.postgres; + +/** + * Write-time flow aggregation math: assigns a flow to fixed time windows and prorates its + * (sampling-scaled) bytes across the windows it spans. This is the ingest-time counterpart to the + * read-time {@link FlowProration}, and is a faithful port of the OpenNMS Nephron streaming + * aggregator ({@code UnalignedFixedWindows} plus {@code Pipeline.bytesInWindow}/{@code aggregatize}), + * reproduced here so an in-process aggregator can run inside Horizon core or Sentinel without the + * Apache Beam / Flink runtime. + * + *

Proration model

+ * A flow is assumed to deliver bytes at a uniform rate over its inclusive interval + * {@code [deltaSwitched, lastSwitched]} (duration {@code last - delta + 1} ms). A window's share is + * computed as the difference of a single monotone cumulative-bytes function evaluated at consecutive + * window boundaries: + * + * 
{@code   C(t) = floor((t - delta + 1) * V / D)      bytesInWindow = C(overlapEnd) - C(overlapStart - 1)}
+ * + * Because consecutive windows share boundaries and {@code C} is one monotone function, the integer + * per-window shares of a flow telescope and sum EXACTLY to {@code floor(V)} (the floored, + * sampling-scaled total) — no bytes are lost or double counted across windows. That + * conservation guarantee is the reason for the cumulative-difference formulation rather than an + * independent per-window {@code V * overlap / D}. + * + *

Window alignment

+ * Windows align to {@code shift + k*windowSize}. Horizon/Sentinel aggregation uses a global grid + * ({@code shift == 0}) so windows are consistent across exporters and cheap to roll up in SQL; + * {@link #perNodeShift(int, long)} reproduces Nephron's per-exporter jitter for an optional + * per-node mode. The window arithmetic assumes {@code timestamp >= shift} (Nephron guards flows with + * {@code deltaSwitched < shift} upstream); with the default {@code shift == 0} this always holds for + * epoch-millisecond timestamps. + */ +final class FixedWindowAggregation { + + private FixedWindowAggregation() { + } + + /** + * Sampling scale-up applied to a flow's byte count. Mirrors {@link FlowProration#effectiveSampling} + * (finite and positive scales; {@code null}/{@code 0}/non-finite are treated as 1) so the write-time + * and read-time paths agree. This is slightly stricter than Nephron, which scales by any + * {@code samplingInterval > 0}. + */ + static double samplingMultiplier(final Double samplingInterval) { + return FlowProration.effectiveSampling(samplingInterval); + } + + /** + * Bytes of a flow attributable to a single window. Port of Nephron's {@code Pipeline.bytesInWindow}. + * + * @param deltaSwitched flow start (inclusive), ms + * @param lastSwitchedInclusive flow end (inclusive), ms + * @param multipliedNumBytes the flow's bytes already scaled by {@link #samplingMultiplier} + * @param windowStart window start (inclusive), ms + * @param windowEndInclusive window end (inclusive), ms (i.e. windowStart + windowSize - 1) + */ + static long bytesInWindow(final long deltaSwitched, + final long lastSwitchedInclusive, + final double multipliedNumBytes, + final long windowStart, + final long windowEndInclusive) { + // The flow duration ranges [delta_switched, last_switched] (both bounds inclusive). + final long flowDurationMs = lastSwitchedInclusive - deltaSwitched + 1; + + // The portion of the flow that falls inside this window (both bounds inclusive). + final long overlapStart = Math.max(deltaSwitched, windowStart); + final long overlapEnd = Math.min(lastSwitchedInclusive, windowEndInclusive); + + // Cumulative bytes delivered through the end of the previous window and through this window's + // end; their difference telescopes so per-window shares sum exactly to floor(multipliedNumBytes). + final long previousEnd = overlapStart - 1; + final long bytesAtPreviousEnd = (long) ((previousEnd - deltaSwitched + 1) * multipliedNumBytes / flowDurationMs); + final long bytesAtEnd = (long) ((overlapEnd - deltaSwitched + 1) * multipliedNumBytes / flowDurationMs); + + return bytesAtEnd - bytesAtPreviousEnd; + } + + /** The number of the (shifted) fixed window a timestamp falls into. Port of {@code UnalignedFixedWindows.windowNumber}. */ + static long windowNumber(final long shift, final long windowSize, final long timestamp) { + return (timestamp - shift) / windowSize; + } + + /** The start (inclusive, ms) of the (shifted) window containing a timestamp. Port of {@code windowStartForTimestamp}. */ + static long windowStartForTimestamp(final long shift, final long windowSize, final long timestamp) { + return timestamp - (timestamp - shift) % windowSize; + } + + /** The start (inclusive, ms) of a window given its number. Port of {@code windowStartForWindowNumber}. */ + static long windowStartForWindowNumber(final long shift, final long windowSize, final long windowNumber) { + return shift + windowNumber * windowSize; + } + + /** + * Per-exporter window shift in {@code [0, windowSize)}, decorrelating window boundaries across + * exporters. Port of {@code UnalignedFixedWindows.perNodeShift}; the {@link #mix(int)} hash is + * fastutil's {@code HashCommon.mix(int)} inlined so no runtime dependency is required. Only needed + * for the optional per-node alignment mode; global-grid aggregation uses {@code shift == 0}. + */ + static long perNodeShift(final int nodeId, final long windowSize) { + return Math.abs(mix(nodeId)) % windowSize; + } + + /** fastutil {@code HashCommon.INT_PHI} (2^32 * (sqrt(5) - 1) / 2), i.e. -1640531527. */ + private static final int INT_PHI = 0x9E3779B9; + + /** fastutil {@code HashCommon.mix(int)}: multiply by INT_PHI, then xor with the high half. */ + private static int mix(final int x) { + final int h = x * INT_PHI; + return h ^ (h >>> 16); + } +} diff --git a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowAggWriter.java b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowAggWriter.java new file mode 100644 index 000000000000..3c1d0f24af5b --- /dev/null +++ b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowAggWriter.java @@ -0,0 +1,101 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.postgres; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.List; +import java.util.Objects; + +import javax.sql.DataSource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.jdbc.core.BatchPreparedStatementSetter; +import org.springframework.jdbc.core.JdbcTemplate; + +/** + * The {@link FlowAggregator} sink that persists closed-window {@link AggregatedFlow} rows to the + * {@code flow_agg} table. Each row carries this writer's {@code writerId}; when several writers feed + * the same window+key (e.g. multiple Sentinels), each writes its own partial row and readers SUM + * them. A single Horizon-core writer therefore produces already-final rows. + */ +public class FlowAggWriter implements java.util.function.Consumer> { + + private static final Logger LOG = LoggerFactory.getLogger(FlowAggWriter.class); + + private static final String INSERT_SQL = + "INSERT INTO flow_agg (window_start, window_end, exporter_node_id, if_index, dscp, dimension, " + + "grouped_by_key, bytes_in, bytes_out, congestion_encountered, non_ecn_capable_transport, " + + "hostname, writer_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + + private final JdbcTemplate jdbcTemplate; + private final String writerId; + + public FlowAggWriter(final DataSource dataSource, final String writerId) { + this.jdbcTemplate = new JdbcTemplate(Objects.requireNonNull(dataSource)); + this.writerId = Objects.requireNonNull(writerId); + } + + @Override + public void accept(final List rows) { + if (rows == null || rows.isEmpty()) { + return; + } + try { + jdbcTemplate.batchUpdate(INSERT_SQL, new BatchPreparedStatementSetter() { + @Override + public void setValues(final PreparedStatement ps, final int i) throws SQLException { + final AggregatedFlow r = rows.get(i); + int c = 1; + // timestamptz columns bound as UTC OffsetDateTime so the stored instant is unambiguous + ps.setObject(c++, Instant.ofEpochMilli(r.windowStartMs).atOffset(ZoneOffset.UTC)); + ps.setObject(c++, Instant.ofEpochMilli(r.windowEndMs).atOffset(ZoneOffset.UTC)); + ps.setInt(c++, r.exporterNodeId); + ps.setInt(c++, r.ifIndex); + if (r.dscp != null) { + ps.setInt(c++, r.dscp); + } else { + ps.setNull(c++, java.sql.Types.SMALLINT); + } + ps.setString(c++, r.dimension.name()); + ps.setString(c++, r.groupedByKey); + ps.setLong(c++, r.bytesIn); + ps.setLong(c++, r.bytesOut); + ps.setBoolean(c++, r.congestionEncountered); + ps.setBoolean(c++, r.nonEcnCapableTransport); + ps.setString(c++, r.hostname); + ps.setString(c, writerId); + } + + @Override + public int getBatchSize() { + return rows.size(); + } + }); + } catch (final Exception e) { + LOG.warn("Failed to persist a batch of {} aggregated flow rows; the batch is dropped.", rows.size(), e); + } + } +} diff --git a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowAggregator.java b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowAggregator.java new file mode 100644 index 000000000000..38700a18c4bd --- /dev/null +++ b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowAggregator.java @@ -0,0 +1,456 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.postgres; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; + +/** + * In-process, write-time flow aggregator: the embeddable, Beam-free counterpart to Nephron. It buffers + * flows into fixed event-time windows keyed by {@code (exporter, interface, dimension, value)}, prorates + * each flow's (sampling-scaled) bytes across the windows it spans via {@link FixedWindowAggregation}, and + * emits one summed {@link AggregatedFlow} per key when a window closes. + * + *

Windowing and watermark

+ * Windows use the global grid ({@code shift == 0}), so every exporter shares the same boundaries and the + * results roll up cleanly in SQL. Event time is the flow's {@code lastSwitched}; the watermark is the + * greatest event time seen. A window {@code [start, start+size)} closes once + * {@code watermark >= start + size + allowedLateness}; {@code allowedLateness} therefore doubles as the + * out-of-order holdback. This phase fires each window exactly once, at close (no early/incremental panes); + * a flow arriving for an already-closed window is dropped and counted. + * + *

Distribution

+ * Each instance aggregates only the flows it is fed. Under multiple writers (e.g. several Sentinels) each + * emits partial rows and the reader sums them per {@code (window, key)} — so no cross-instance state is + * needed. A single Horizon-core instance is simply the one-writer case, where a row is already final. + * + *

Scope (phase 2)

+ * Sum-by-key over interface totals plus application/conversation/host; no TOS dimension and no top-K + * capping yet (those are the next phase). Fault tolerance is bounded by the concession that a restart + * loses the still-open windows held in memory. + */ +public class FlowAggregator implements AutoCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(FlowAggregator.class); + + /** Dimensions capped by top-K; the long tail is rolled into one null-key "Other" row per window. */ + private static final Set CAPPED = EnumSet.of( + AggregatedFlow.Dimension.APPLICATION, + AggregatedFlow.Dimension.CONVERSATION, + AggregatedFlow.Dimension.HOST); + + /** Order for top-K selection: most bytes first, then key ascending for deterministic ties. */ + private static final Comparator> BY_BYTES_DESC = (x, y) -> { + final int c = Long.compare(y.getValue().bytes(), x.getValue().bytes()); + if (c != 0) { + return c; + } + final String kx = x.getKey().groupedByKey; + final String ky = y.getKey().groupedByKey; + if (kx == null) { + return ky == null ? 0 : 1; + } + return ky == null ? -1 : kx.compareTo(ky); + }; + + private final long windowSizeMs; + private final long allowedLatenessMs; + private final long flushIntervalMs; + private final long idleFlushMs; + private final int topK; + private final Consumer> sink; + + /** Open windows, keyed by window start (ascending), each holding its per-key accumulators. */ + private final ConcurrentSkipListMap> windows = new ConcurrentSkipListMap<>(); + /** Wall-clock time each open window was first created, for the idle (processing-time) flush fallback. */ + private final ConcurrentHashMap windowFirstSeenMs = new ConcurrentHashMap<>(); + private final AtomicLong maxEventTimeMs = new AtomicLong(Long.MIN_VALUE); + + /** Wall clock, injectable for tests. */ + private java.util.function.LongSupplier clock = System::currentTimeMillis; + + private final Meter flowsAggregated; + private final Meter flowsDroppedLate; + private final Meter rowsEmitted; + + private volatile boolean running = false; + private Thread flusher; + + /** + * @param topK per (exporter, interface, dimension) cap for the high-cardinality dimensions + * (application/conversation/host); the remainder is rolled into one "Other" row. + * {@code <= 0} disables capping (emit every key — interface totals are never capped). + * @param idleFlushMs wall-clock age after which an open window is flushed even if the event-time + * watermark has stalled (e.g. an exporter went quiet). {@code <= 0} auto-derives a value + * safely larger than the normal event-time close ({@code windowSize + lateness + 60s}). + */ + public FlowAggregator(final long windowSizeMs, final long allowedLatenessMs, final long flushIntervalMs, + final int topK, final long idleFlushMs, final Consumer> sink, + final MetricRegistry metrics) { + if (windowSizeMs < 1) { + throw new IllegalArgumentException("windowSizeMs must be >= 1"); + } + if (allowedLatenessMs < 0) { + throw new IllegalArgumentException("allowedLatenessMs must be >= 0"); + } + this.windowSizeMs = windowSizeMs; + this.allowedLatenessMs = allowedLatenessMs; + this.flushIntervalMs = flushIntervalMs > 0 ? flushIntervalMs : 1000; + this.idleFlushMs = idleFlushMs > 0 ? idleFlushMs : windowSizeMs + allowedLatenessMs + 60_000L; + this.topK = topK; + this.sink = Objects.requireNonNull(sink); + Objects.requireNonNull(metrics); + this.flowsAggregated = metrics.meter(MetricRegistry.name("flowAggregator", "flowsAggregated")); + this.flowsDroppedLate = metrics.meter(MetricRegistry.name("flowAggregator", "flowsDroppedLate")); + this.rowsEmitted = metrics.meter(MetricRegistry.name("flowAggregator", "rowsEmitted")); + metrics.register(MetricRegistry.name("flowAggregator", "openWindows"), + (com.codahale.metrics.Gauge) windows::size); + } + + public synchronized void start() { + if (running) { + return; + } + running = true; + flusher = new Thread(this::flushLoop, "postgres-flow-aggregator"); + flusher.setDaemon(true); + flusher.start(); + LOG.info("FlowAggregator started (windowSizeMs={}, allowedLatenessMs={}, flushIntervalMs={}, topK={}, idleFlushMs={}).", + windowSizeMs, allowedLatenessMs, flushIntervalMs, topK, idleFlushMs); + } + + /** Aggregate a flow. Returns {@code false} (and counts it) when the flow is unusable or wholly late. */ + boolean add(final FlowInput f) { + if (f == null) { + return false; + } + // Decide lateness against the watermark established by PRIOR flows, then advance it. A flow must + // not close its own earlier windows just because it also extends into a later one. + final long closedAtOrBefore = closedWindowStartCeiling(maxEventTimeMs.get()); + maxEventTimeMs.updateAndGet(prev -> Math.max(prev, f.lastSwitchedMs)); + + final double value = f.bytes * FixedWindowAggregation.samplingMultiplier(f.samplingInterval); + final long firstWindow = FixedWindowAggregation.windowNumber(0L, windowSizeMs, f.deltaSwitchedMs); + final long lastWindow = FixedWindowAggregation.windowNumber(0L, windowSizeMs, f.lastSwitchedMs); + + boolean anyAccepted = false; + for (long wn = firstWindow; wn <= lastWindow; wn++) { + final long windowStart = wn * windowSizeMs; // shift == 0 + if (windowStart <= closedAtOrBefore) { + // The window has already closed (and likely flushed); do not resurrect it. + flowsDroppedLate.mark(); + continue; + } + final long windowEndInclusive = windowStart + windowSizeMs - 1; + final long bytes = FixedWindowAggregation.bytesInWindow( + f.deltaSwitchedMs, f.lastSwitchedMs, value, windowStart, windowEndInclusive); + if (bytes <= 0) { + continue; // no bytes attributable to this window (e.g. sub-millisecond overlap rounding) + } + final ConcurrentHashMap w = windows.computeIfAbsent(windowStart, k -> new ConcurrentHashMap<>()); + windowFirstSeenMs.putIfAbsent(windowStart, clock.getAsLong()); // for the idle-flush fallback + accumulateDimensions(w, f, bytes, null); // without-TOS rollup (over all DSCP) + if (f.dscp != null) { + accumulateDimensions(w, f, bytes, f.dscp); // with-TOS (this flow's DSCP) + } + anyAccepted = true; + } + if (anyAccepted) { + flowsAggregated.mark(); + } + return anyAccepted; + } + + /** Convenience adapter for the enriched-flow API; skips flows missing required fields. */ + boolean add(final org.opennms.integration.api.v1.flows.Flow flow) { + return add(FlowInput.from(flow)); + } + + /** Accumulate every dimension for one DSCP scope ({@code dscp == null} is the without-TOS rollup). */ + private static void accumulateDimensions(final Map w, final FlowInput f, final long bytes, final Integer dscp) { + accumulate(w, f.exporterNodeId, f.ifIndex, dscp, AggregatedFlow.Dimension.INTERFACE, null, bytes, f.ingress, f.ecn, null); + if (f.application != null) { + accumulate(w, f.exporterNodeId, f.ifIndex, dscp, AggregatedFlow.Dimension.APPLICATION, f.application, bytes, f.ingress, f.ecn, null); + } + if (f.convoKey != null) { + accumulate(w, f.exporterNodeId, f.ifIndex, dscp, AggregatedFlow.Dimension.CONVERSATION, f.convoKey, bytes, f.ingress, f.ecn, null); + } + if (f.srcAddr != null) { + accumulate(w, f.exporterNodeId, f.ifIndex, dscp, AggregatedFlow.Dimension.HOST, f.srcAddr, bytes, f.ingress, f.ecn, f.srcHostname); + } + if (f.dstAddr != null) { + accumulate(w, f.exporterNodeId, f.ifIndex, dscp, AggregatedFlow.Dimension.HOST, f.dstAddr, bytes, f.ingress, f.ecn, f.dstHostname); + } + } + + private static void accumulate(final Map window, final int exporterNodeId, final int ifIndex, + final Integer dscp, final AggregatedFlow.Dimension dimension, final String groupedByKey, + final long bytes, final boolean ingress, final Integer ecn, final String hostname) { + window.computeIfAbsent(new Key(exporterNodeId, ifIndex, dscp, dimension, groupedByKey), k -> new Acc()) + .add(bytes, ingress, ecn, hostname); + } + + /** The greatest window-start that is considered closed given a watermark, or {@code Long.MIN_VALUE} if none. */ + private long closedWindowStartCeiling(final long watermark) { + if (watermark == Long.MIN_VALUE) { + return Long.MIN_VALUE; + } + // window [s, s+size) closed when watermark >= s + size + lateness, i.e. s <= watermark - size - lateness + return watermark - windowSizeMs - allowedLatenessMs; + } + + private void flushLoop() { + while (running) { + try { + Thread.sleep(flushIntervalMs); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + try { + flushClosedWindows(); + flushIdleWindows(); + } catch (final Exception e) { + LOG.warn("Flow aggregation flush failed; will retry on the next tick.", e); + } + } + } + + /** Emit and evict every window that has closed under the current watermark. */ + void flushClosedWindows() { + final long threshold = closedWindowStartCeiling(maxEventTimeMs.get()); + drainWindowsUpTo(threshold); + } + + /** + * Processing-time fallback: emit and evict windows that have been open longer than {@code idleFlushMs} + * in wall-clock time, regardless of the event-time watermark. This releases windows whose exporter has + * gone quiet (so the watermark no longer advances to close them). + */ + void flushIdleWindows() { + final long cutoff = clock.getAsLong() - idleFlushMs; + final List batch = new ArrayList<>(); + for (final Map.Entry e : windowFirstSeenMs.entrySet()) { + if (e.getValue() <= cutoff) { + final Long windowStart = e.getKey(); + windowFirstSeenMs.remove(windowStart); + final ConcurrentHashMap acc = windows.remove(windowStart); + if (acc != null) { + emitWindow(windowStart, acc, batch); + } + } + } + if (!batch.isEmpty()) { + rowsEmitted.mark(batch.size()); + sink.accept(batch); + } + } + + private void drainWindowsUpTo(final long thresholdInclusive) { + final List batch = new ArrayList<>(); + while (!windows.isEmpty()) { + final Long first = windows.firstKey(); + if (first == null || first > thresholdInclusive) { + break; + } + final Map.Entry> entry = windows.pollFirstEntry(); + if (entry != null) { + windowFirstSeenMs.remove(entry.getKey()); + emitWindow(entry.getKey(), entry.getValue(), batch); + } + } + if (!batch.isEmpty()) { + rowsEmitted.mark(batch.size()); + sink.accept(batch); + } + } + + /** Test hook: override the wall clock used for the idle flush. */ + void setClock(final java.util.function.LongSupplier clock) { + this.clock = Objects.requireNonNull(clock); + } + + private void emitWindow(final long windowStart, final Map accumulators, final List out) { + final long windowEnd = windowStart + windowSizeMs; + if (topK <= 0) { + for (final Map.Entry e : accumulators.entrySet()) { + emit(out, windowStart, windowEnd, e.getKey(), e.getValue()); + } + return; + } + // Group entries by outer key (exporter, interface, dimension) so top-K is applied per parent. + final Map>> byOuter = new HashMap<>(); + for (final Map.Entry e : accumulators.entrySet()) { + final Key k = e.getKey(); + byOuter.computeIfAbsent(new Key(k.exporterNodeId, k.ifIndex, k.dscp, k.dimension, null), x -> new ArrayList<>()).add(e); + } + for (final Map.Entry>> group : byOuter.entrySet()) { + final Key outer = group.getKey(); + final List> entries = group.getValue(); + if (!CAPPED.contains(outer.dimension)) { + // Interface (and future exporter/tos) totals are never capped. + for (final Map.Entry e : entries) { + emit(out, windowStart, windowEnd, e.getKey(), e.getValue()); + } + continue; + } + // Keep the top-K by bytes; roll everything else into one null-key "Other" row. + entries.sort(BY_BYTES_DESC); + Acc other = null; + for (int i = 0; i < entries.size(); i++) { + if (i < topK) { + emit(out, windowStart, windowEnd, entries.get(i).getKey(), entries.get(i).getValue()); + } else { + if (other == null) { + other = new Acc(); + } + other.mergeCounts(entries.get(i).getValue()); + } + } + if (other != null) { + out.add(new AggregatedFlow(windowStart, windowEnd, outer.exporterNodeId, outer.ifIndex, + outer.dscp, outer.dimension, null, other.bytesIn, other.bytesOut, + other.congestionEncountered, other.nonEcnCapableTransport, null)); + } + } + } + + private static void emit(final List out, final long windowStart, final long windowEnd, + final Key k, final Acc a) { + out.add(new AggregatedFlow(windowStart, windowEnd, k.exporterNodeId, k.ifIndex, k.dscp, k.dimension, + k.groupedByKey, a.bytesIn, a.bytesOut, a.congestionEncountered, a.nonEcnCapableTransport, a.hostname)); + } + + @Override + public synchronized void close() { + running = false; + if (flusher != null) { + flusher.interrupt(); + try { + flusher.join(TimeUnit.SECONDS.toMillis(30)); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + // Best-effort final flush of everything still buffered, regardless of the watermark. + drainWindowsUpTo(Long.MAX_VALUE); + LOG.info("FlowAggregator stopped."); + } + + /** Immutable grouping key within a window ({@code dscp == null} is the without-TOS scope). */ + private static final class Key { + private final int exporterNodeId; + private final int ifIndex; + private final Integer dscp; + private final AggregatedFlow.Dimension dimension; + private final String groupedByKey; + + Key(final int exporterNodeId, final int ifIndex, final Integer dscp, + final AggregatedFlow.Dimension dimension, final String groupedByKey) { + this.exporterNodeId = exporterNodeId; + this.ifIndex = ifIndex; + this.dscp = dscp; + this.dimension = dimension; + this.groupedByKey = groupedByKey; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Key)) { + return false; + } + final Key key = (Key) o; + return exporterNodeId == key.exporterNodeId && ifIndex == key.ifIndex + && Objects.equals(dscp, key.dscp) && dimension == key.dimension + && Objects.equals(groupedByKey, key.groupedByKey); + } + + @Override + public int hashCode() { + return Objects.hash(exporterNodeId, ifIndex, dscp, dimension, groupedByKey); + } + } + + /** Mutable per-key accumulator. Mirrors Nephron's {@code Aggregate} merge semantics. */ + private static final class Acc { + private long bytesIn; + private long bytesOut; + private boolean congestionEncountered; + private boolean nonEcnCapableTransport; + private String hostname; + + void add(final long bytes, final boolean ingress, final Integer ecn, final String hn) { + if (ingress) { + bytesIn += bytes; + } else { + bytesOut += bytes; + } + // ECN: congestion when any record has CE (3); non-ECT (0) or an absent value both flag non-ECT. + if (ecn != null) { + if (ecn == 3) { + congestionEncountered = true; + } + if (ecn == 0) { + nonEcnCapableTransport = true; + } + } else { + nonEcnCapableTransport = true; + } + // Deterministic host name: keep the lexicographically smaller non-empty value. + if (hn != null && !hn.isEmpty() && (hostname == null || hn.compareTo(hostname) < 0)) { + hostname = hn; + } + } + + long bytes() { + return bytesIn + bytesOut; + } + + /** Fold another accumulator's byte counts and ECN flags into this one (for the "Other" bucket). */ + void mergeCounts(final Acc o) { + bytesIn += o.bytesIn; + bytesOut += o.bytesOut; + congestionEncountered |= o.congestionEncountered; + nonEcnCapableTransport |= o.nonEcnCapableTransport; + } + } +} diff --git a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowInput.java b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowInput.java new file mode 100644 index 000000000000..5c5f486801f5 --- /dev/null +++ b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/FlowInput.java @@ -0,0 +1,109 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.postgres; + +import java.time.Instant; + +import org.opennms.integration.api.v1.flows.Flow; + +/** + * The subset of an enriched {@link Flow} the write-time aggregator needs, extracted into a small + * immutable holder. Keeping the engine ({@link FlowAggregator}) decoupled from the (large) {@code Flow} + * interface makes the aggregation/windowing logic unit-testable without mocking dozens of accessors, + * and isolates the one piece of interpretation that matters: the aggregation interface is the flow's + * ingress interface for INGRESS flows and its egress interface otherwise (mirroring + * Nephron's {@code RefType.INTERFACE_PART}). + */ +final class FlowInput { + + final long deltaSwitchedMs; + final long lastSwitchedMs; + final long bytes; + final Double samplingInterval; + final boolean ingress; + final int exporterNodeId; + final int ifIndex; + final String application; + final String convoKey; + final String srcAddr; + final String dstAddr; + final String srcHostname; + final String dstHostname; + final Integer ecn; + final Integer dscp; + + FlowInput(final long deltaSwitchedMs, final long lastSwitchedMs, final long bytes, + final Double samplingInterval, final boolean ingress, final int exporterNodeId, final int ifIndex, + final String application, final String convoKey, final String srcAddr, final String dstAddr, + final String srcHostname, final String dstHostname, final Integer ecn, final Integer dscp) { + this.deltaSwitchedMs = deltaSwitchedMs; + this.lastSwitchedMs = lastSwitchedMs; + this.bytes = bytes; + this.samplingInterval = samplingInterval; + this.ingress = ingress; + this.exporterNodeId = exporterNodeId; + this.ifIndex = ifIndex; + this.application = application; + this.convoKey = convoKey; + this.srcAddr = srcAddr; + this.dstAddr = dstAddr; + this.srcHostname = srcHostname; + this.dstHostname = dstHostname; + this.ecn = ecn; + this.dscp = dscp; + } + + /** + * Extract the aggregation inputs from a {@link Flow}. Returns {@code null} when the flow lacks the + * fields the aggregator requires (switched timestamps, exporter node, the direction's interface) so + * the caller can count and skip it rather than aggregate garbage. + */ + static FlowInput from(final Flow flow) { + final Instant delta = flow.getDeltaSwitched(); + final Instant last = flow.getLastSwitched(); + final Flow.NodeInfo exporter = flow.getExporterNodeInfo(); + if (delta == null || last == null || exporter == null) { + return null; + } + final boolean ingress = flow.getDirection() != null && "INGRESS".equals(flow.getDirection().name()); + final Integer ifIndex = ingress ? flow.getInputSnmp() : flow.getOutputSnmp(); + if (ifIndex == null) { + return null; + } + return new FlowInput( + delta.toEpochMilli(), + last.toEpochMilli(), + flow.getBytes(), + flow.getSamplingInterval(), + ingress, + exporter.getNodeId(), + ifIndex, + flow.getApplication(), + flow.getConvoKey(), + flow.getSrcAddr(), + flow.getDstAddr(), + flow.getSrcAddrHostname().orElse(null), + flow.getDstAddrHostname().orElse(null), + flow.getEcn(), + flow.getDscp()); + } +} diff --git a/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresSmartQueryService.java b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresSmartQueryService.java new file mode 100644 index 000000000000..23ac9b8df173 --- /dev/null +++ b/features/flows/postgres/src/main/java/org/opennms/netmgt/flows/postgres/PostgresSmartQueryService.java @@ -0,0 +1,258 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.postgres; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +import org.opennms.netmgt.flows.api.Conversation; +import org.opennms.netmgt.flows.api.Directional; +import org.opennms.netmgt.flows.api.FlowQueryService; +import org.opennms.netmgt.flows.api.Host; +import org.opennms.netmgt.flows.api.LimitedCardinalityField; +import org.opennms.netmgt.flows.api.TrafficSummary; +import org.opennms.netmgt.flows.filter.api.Filter; +import org.opennms.netmgt.flows.filter.api.TimeRangeFilter; + +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; +import com.google.common.collect.Table; + +/** + * Routes each flow query to either the raw ({@link PostgresFlowQueryService}) or aggregated + * ({@link AggregatedFlowQueryService}) backend, port of the Elasticsearch {@code SmartQueryService}. + * + *

Routing (identical to ES): the override flags win first ({@code alwaysUseRawForQueries} defaults + * true, so aggregates are opt-in); queries for specific entities (explicit sets / entity listings) always + * go raw; a {@link TimeRangeFilter} is required for aggregates; then a query goes aggregated when its + * range duration is at least {@code timeRangeDurationAggregateThresholdMs} (default 2 min) or its endpoint + * is older than {@code timeRangeEndpointAggregateThresholdMs} (default 7 days) — otherwise raw. + */ +public class PostgresSmartQueryService implements FlowQueryService { + + private final FlowQueryService rawQueryService; + private final FlowQueryService aggQueryService; + + public enum QueryServiceType { RAW, AGG } + + private boolean alwaysUseAggForQueries = false; + private boolean alwaysUseRawForQueries = true; + + private long timeRangeDurationAggregateThresholdMs = TimeUnit.MINUTES.toMillis(2); + private long timeRangeEndpointAggregateThresholdMs = TimeUnit.DAYS.toMillis(7); + + private final Timer rawQuerySuccessTimer; + private final Timer rawQueryFailureTimer; + private final Timer aggregatedQuerySuccessTimer; + private final Timer aggregatedQueryFailureTimer; + + public PostgresSmartQueryService(final MetricRegistry metricRegistry, final FlowQueryService rawQueryService, + final FlowQueryService aggQueryService) { + this.rawQueryService = Objects.requireNonNull(rawQueryService); + this.aggQueryService = Objects.requireNonNull(aggQueryService); + this.rawQuerySuccessTimer = metricRegistry.timer("rawQuerySuccess"); + this.rawQueryFailureTimer = metricRegistry.timer("rawQueryFailure"); + this.aggregatedQuerySuccessTimer = metricRegistry.timer("aggregatedQuerySuccess"); + this.aggregatedQueryFailureTimer = metricRegistry.timer("aggregatedQueryFailure"); + } + + QueryServiceType getDelegate(final List filters, final boolean isQueryForSpecificEntities) { + if (alwaysUseRawForQueries) { + return QueryServiceType.RAW; + } else if (alwaysUseAggForQueries) { + return QueryServiceType.AGG; + } + // Specific-entity queries are not supported by the aggregate backend. + if (isQueryForSpecificEntities) { + return QueryServiceType.RAW; + } + final Optional timeRangeFilter = Filter.find(filters, TimeRangeFilter.class); + if (!timeRangeFilter.isPresent()) { + return QueryServiceType.RAW; + } + if (timeRangeFilter.get().getDurationMs() >= timeRangeDurationAggregateThresholdMs) { + return QueryServiceType.AGG; + } + if ((System.currentTimeMillis() - timeRangeFilter.get().getEnd()) > timeRangeEndpointAggregateThresholdMs) { + return QueryServiceType.AGG; + } + return QueryServiceType.RAW; + } + + private CompletableFuture runWithDelegate(final List filters, final boolean isQueryForSpecificEntities, + final Function> query) { + switch (getDelegate(filters, isQueryForSpecificEntities)) { + case AGG: + return timeAsync(aggregatedQuerySuccessTimer, aggregatedQueryFailureTimer, () -> query.apply(aggQueryService)); + case RAW: + default: + return timeAsync(rawQuerySuccessTimer, rawQueryFailureTimer, () -> query.apply(rawQueryService)); + } + } + + @Override + public CompletableFuture getFlowCount(final List filters) { + return runWithDelegate(filters, false, qs -> qs.getFlowCount(filters)); + } + + @Override + public CompletableFuture> getApplications(final String matchingPrefix, final long limit, final List filters) { + return runWithDelegate(filters, true, qs -> qs.getApplications(matchingPrefix, limit, filters)); + } + + @Override + public CompletableFuture>> getTopNApplicationSummaries(final int n, final boolean includeOther, final List filters) { + return runWithDelegate(filters, false, qs -> qs.getTopNApplicationSummaries(n, includeOther, filters)); + } + + @Override + public CompletableFuture>> getApplicationSummaries(final Set applications, final boolean includeOther, final List filters) { + return runWithDelegate(filters, true, qs -> qs.getApplicationSummaries(applications, includeOther, filters)); + } + + @Override + public CompletableFuture, Long, Double>> getApplicationSeries(final Set applications, final long step, final boolean includeOther, final List filters) { + return runWithDelegate(filters, true, qs -> qs.getApplicationSeries(applications, step, includeOther, filters)); + } + + @Override + public CompletableFuture, Long, Double>> getTopNApplicationSeries(final int n, final long step, final boolean includeOther, final List filters) { + return runWithDelegate(filters, false, qs -> qs.getTopNApplicationSeries(n, step, includeOther, filters)); + } + + @Override + public CompletableFuture> getConversations(final String locationPattern, final String protocolPattern, final String lowerIPPattern, final String upperIPPattern, final String applicationPattern, final long limit, final List filters) { + return runWithDelegate(filters, true, qs -> qs.getConversations(locationPattern, protocolPattern, lowerIPPattern, upperIPPattern, applicationPattern, limit, filters)); + } + + @Override + public CompletableFuture>> getTopNConversationSummaries(final int n, final boolean includeOther, final List filters) { + return runWithDelegate(filters, false, qs -> qs.getTopNConversationSummaries(n, includeOther, filters)); + } + + @Override + public CompletableFuture>> getConversationSummaries(final Set conversations, final boolean includeOther, final List filters) { + return runWithDelegate(filters, true, qs -> qs.getConversationSummaries(conversations, includeOther, filters)); + } + + @Override + public CompletableFuture, Long, Double>> getConversationSeries(final Set conversations, final long step, final boolean includeOther, final List filters) { + return runWithDelegate(filters, true, qs -> qs.getConversationSeries(conversations, step, includeOther, filters)); + } + + @Override + public CompletableFuture, Long, Double>> getTopNConversationSeries(final int n, final long step, final boolean includeOther, final List filters) { + return runWithDelegate(filters, false, qs -> qs.getTopNConversationSeries(n, step, includeOther, filters)); + } + + @Override + public CompletableFuture> getHosts(final String regex, final long limit, final List filters) { + return runWithDelegate(filters, true, qs -> qs.getHosts(regex, limit, filters)); + } + + @Override + public CompletableFuture>> getTopNHostSummaries(final int n, final boolean includeOther, final List filters) { + return runWithDelegate(filters, false, qs -> qs.getTopNHostSummaries(n, includeOther, filters)); + } + + @Override + public CompletableFuture>> getHostSummaries(final Set hosts, final boolean includeOther, final List filters) { + return runWithDelegate(filters, true, qs -> qs.getHostSummaries(hosts, includeOther, filters)); + } + + @Override + public CompletableFuture, Long, Double>> getHostSeries(final Set hosts, final long step, final boolean includeOther, final List filters) { + return runWithDelegate(filters, true, qs -> qs.getHostSeries(hosts, step, includeOther, filters)); + } + + @Override + public CompletableFuture, Long, Double>> getTopNHostSeries(final int n, final long step, final boolean includeOther, final List filters) { + return runWithDelegate(filters, false, qs -> qs.getTopNHostSeries(n, step, includeOther, filters)); + } + + @Override + public CompletableFuture> getFieldValues(final LimitedCardinalityField field, final List filters) { + return runWithDelegate(filters, false, qs -> qs.getFieldValues(field, filters)); + } + + @Override + public CompletableFuture>> getFieldSummaries(final LimitedCardinalityField field, final List filters) { + return runWithDelegate(filters, false, qs -> qs.getFieldSummaries(field, filters)); + } + + @Override + public CompletableFuture, Long, Double>> getFieldSeries(final LimitedCardinalityField field, final long step, final List filters) { + return runWithDelegate(filters, false, qs -> qs.getFieldSeries(field, step, filters)); + } + + // --- config setters (blueprint) --- + public boolean isAlwaysUseAggForQueries() { return alwaysUseAggForQueries; } + + public void setAlwaysUseAggForQueries(final boolean alwaysUseAggForQueries) { + this.alwaysUseAggForQueries = alwaysUseAggForQueries; + if (alwaysUseAggForQueries) { + this.alwaysUseRawForQueries = false; + } + } + + public boolean isAlwaysUseRawForQueries() { return alwaysUseRawForQueries; } + + public void setAlwaysUseRawForQueries(final boolean alwaysUseRawForQueries) { + this.alwaysUseRawForQueries = alwaysUseRawForQueries; + if (alwaysUseRawForQueries) { + this.alwaysUseAggForQueries = false; + } + } + + public void setTimeRangeDurationAggregateThresholdMs(final long v) { this.timeRangeDurationAggregateThresholdMs = v; } + public void setTimeRangeEndpointAggregateThresholdMs(final long v) { this.timeRangeEndpointAggregateThresholdMs = v; } + + private static CompletableFuture timeAsync(final Timer successTimer, final Timer failureTimer, + final Callable> operation) { + final Timer.Context successContext = successTimer.time(); + final Timer.Context failureContext = failureTimer.time(); + try { + final CompletableFuture promise = new CompletableFuture<>(); + final CompletableFuture future = operation.call(); + future.handleAsync((success, failure) -> { + if (failure == null) { + successContext.stop(); + promise.complete(success); + } else { + failureContext.stop(); + promise.completeExceptionally(failure); + } + return null; + }); + return promise; + } catch (final Exception ex) { + failureContext.stop(); + throw new RuntimeException(ex); + } + } +} diff --git a/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml index 469993b703bb..4eaa9a158877 100644 --- a/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ b/features/flows/postgres/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@ -39,6 +39,30 @@ + + + + + + + + + + + + + + + + @@ -117,7 +141,30 @@ - + + + + + + + + + + + + + + + + + + + @@ -131,4 +178,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/features/flows/postgres/src/main/resources/org/opennms/netmgt/flows/postgres/changelog.xml b/features/flows/postgres/src/main/resources/org/opennms/netmgt/flows/postgres/changelog.xml index 21ad9b9db7dd..e0ed4a042985 100644 --- a/features/flows/postgres/src/main/resources/org/opennms/netmgt/flows/postgres/changelog.xml +++ b/features/flows/postgres/src/main/resources/org/opennms/netmgt/flows/postgres/changelog.xml @@ -18,8 +18,15 @@ --> + + ANY - + SELECT count(*) FROM pg_class WHERE relname = 'flow' AND relkind IN ('r','p') - + retention_days))::date; - child record; - child_date date; + step interval; + fmt text; + bucket timestamptz; + part_name text; + cutoff timestamptz := now() - retention; + child record; + upper_txt text; + upper_bound timestamptz; BEGIN - -- Pre-create today + upcoming daily partitions. - FOR i IN 0..premake_days LOOP - d := (now()::date + i); - part_name := format('flow_p%s', to_char(d, 'YYYYMMDD')); + -- No-op if the parent table does not exist (e.g. flow_agg before aggregation is installed). + IF NOT EXISTS (SELECT 1 FROM pg_class WHERE relname = parent AND relkind = 'p') THEN + RETURN; + END IF; + + IF granularity NOT IN ('hour', 'day', 'week', 'month') THEN + RAISE EXCEPTION 'flow_maintain_partitions: unsupported granularity % (use hour|day|week|month)', granularity; + END IF; + step := ('1 ' || granularity)::interval; + fmt := CASE granularity + WHEN 'hour' THEN '"p"YYYYMMDD"h"HH24' + WHEN 'day' THEN '"p"YYYYMMDD' + WHEN 'week' THEN '"p"IYYY"w"IW' + WHEN 'month' THEN '"p"YYYYMM' + END; + + -- Pre-create the current + `premake` future buckets. + FOR i IN 0..premake LOOP + bucket := date_trunc(granularity, now()) + (i * step); + part_name := parent || '_' || to_char(bucket, fmt); IF NOT EXISTS (SELECT 1 FROM pg_class WHERE relname = part_name) THEN BEGIN - EXECUTE format( - 'CREATE TABLE %I PARTITION OF flow FOR VALUES FROM (%L) TO (%L)', - part_name, d::timestamptz, (d + 1)::timestamptz); + EXECUTE format('CREATE TABLE %I PARTITION OF %I FOR VALUES FROM (%L) TO (%L)', + part_name, parent, bucket, bucket + step); EXCEPTION WHEN others THEN + -- Overlapping/pre-existing range (e.g. a coarser legacy partition still covers it). RAISE NOTICE 'flow_maintain_partitions: could not create %: %', part_name, SQLERRM; END; END IF; END LOOP; - -- Drop partitions whose day is older than the retention cutoff. + -- Drop partitions whose entire range is older than the retention cutoff. FOR child IN - SELECT c.relname AS relname + SELECT c.oid, c.relname, pg_get_expr(c.relpartbound, c.oid) AS bound FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid JOIN pg_class p ON p.oid = i.inhparent - WHERE p.relname = 'flow' AND c.relname ~ '^flow_p[0-9]{8}$' + WHERE p.relname = parent LOOP - child_date := to_date(right(child.relname, 8), 'YYYYMMDD'); - IF child_date < cutoff THEN + -- Upper bound of a range partition: FOR VALUES FROM (...) TO (''); DEFAULT has no match. + upper_txt := (regexp_match(child.bound, 'TO \(''([^'']+)''\)'))[1]; + CONTINUE WHEN upper_txt IS NULL; + upper_bound := upper_txt::timestamptz; + IF upper_bound <= cutoff THEN EXECUTE format('DROP TABLE IF EXISTS %I', child.relname); END IF; END LOOP; END; $$ LANGUAGE plpgsql; ]]> - DROP FUNCTION IF EXISTS flow_maintain_partitions(int, int); + DROP FUNCTION IF EXISTS flow_maintain_partitions(text, text, text, interval, int); - + + ANY + + + + + + ANY + + SELECT count(*) FROM pg_class WHERE relname = 'flow_agg' AND relkind IN ('r','p') + + + DROP TABLE IF EXISTS flow_agg; + + + + + ANY + + + + + + diff --git a/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/FixedWindowAggregationTest.java b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/FixedWindowAggregationTest.java new file mode 100644 index 000000000000..f8153f463cf6 --- /dev/null +++ b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/FixedWindowAggregationTest.java @@ -0,0 +1,222 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.postgres; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Random; + +import org.junit.Test; + +/** + * Verifies the write-time aggregation math ported from the OpenNMS Nephron streaming aggregator into + * {@link FixedWindowAggregation}. + * + *

The two properties that actually matter are proven directly rather than by re-deriving the + * ported formula (which would be tautological): + *

    + *
  • Conservation — splitting a flow across the windows it spans yields exactly the + * same total as not splitting it (the telescoping guarantee), and that total is the floored, + * sampling-scaled byte count.
    • + *
  • Fidelity — each window's integer share stays within one byte of the ideal + * real-valued proration {@code V * overlap / duration} (an independent formula).
    • + *
+ * The windowing invariants mirror Nephron's {@code UnalignedFixedWindowsTest}, and + * {@link #perNodeShiftMatchesFastutilGoldenValues()} locks the inlined hash to values computed from + * fastutil 8.5.2 (the version Nephron pins). + */ +public class FixedWindowAggregationTest { + + // ---- explicit proration cases ------------------------------------------------------------- + + @Test + public void flowFullyInsideOneWindowGetsAllBytes() { + // window [1000, 2000), flow [1200, 1700], 1000 bytes, no sampling + assertEquals(1000L, FixedWindowAggregation.bytesInWindow(1200L, 1700L, 1000.0, 1000L, 1999L)); + } + + @Test + public void flowEvenlySpanningTwoWindowsSplitsInHalf() { + // 2000 bytes uniformly over [1000, 2999] (2000 ms) across two 1000 ms windows + final long a = FixedWindowAggregation.bytesInWindow(1000L, 2999L, 2000.0, 1000L, 1999L); + final long b = FixedWindowAggregation.bytesInWindow(1000L, 2999L, 2000.0, 2000L, 2999L); + assertEquals(1000L, a); + assertEquals(1000L, b); + assertEquals(2000L, a + b); + } + + @Test + public void zeroDurationFlowLandsWhollyInItsWindow() { + // delta == last: whole value goes to the single window containing the instant + assertEquals(1000L, FixedWindowAggregation.bytesInWindow(1500L, 1500L, 1000.0, 1000L, 1999L)); + } + + @Test + public void samplingMultiplierScalesBytesAndGuardsDegenerateValues() { + assertEquals(10.0, FixedWindowAggregation.samplingMultiplier(10.0), 0.0); + assertEquals(1.0, FixedWindowAggregation.samplingMultiplier(0.0), 0.0); + assertEquals(1.0, FixedWindowAggregation.samplingMultiplier(null), 0.0); + assertEquals(1.0, FixedWindowAggregation.samplingMultiplier(Double.NaN), 0.0); + assertEquals(1.0, FixedWindowAggregation.samplingMultiplier(Double.POSITIVE_INFINITY), 0.0); + // 1000 bytes at sampling 10 -> 10000 attributed to the containing window + final double v = 1000L * FixedWindowAggregation.samplingMultiplier(10.0); + assertEquals(10000L, FixedWindowAggregation.bytesInWindow(1200L, 1700L, v, 1000L, 1999L)); + } + + // ---- conservation (the telescoping guarantee) --------------------------------------------- + + /** + * Splitting a flow across the windows it spans must total exactly the same as computing it over a + * single window covering the whole flow. This exercises the window tiling and the telescoping + * cumulative-difference arithmetic together, and holds exactly regardless of floating-point + * rounding (adjacent windows evaluate the cumulative function at an identical boundary). + */ + @Test + public void splittingAcrossWindowsConservesBytes() { + final Random rnd = new Random(20240723L); + for (int i = 0; i < 200000; i++) { + final long windowSize = 1 + (long) rnd.nextInt(100000); + final long shift = rnd.nextBoolean() ? 0L : FixedWindowAggregation.perNodeShift(rnd.nextInt(100000), windowSize); + final long delta = 1_000_000_000L + (long) rnd.nextInt(1_000_000); + final long last = delta + (long) rnd.nextInt(1_000_000); + final double v = (1 + rnd.nextInt(1_000_000)) * randomSampling(rnd); + + final long whole = FixedWindowAggregation.bytesInWindow(delta, last, v, delta, last); + + final long firstWindow = FixedWindowAggregation.windowNumber(shift, windowSize, delta); + final long lastWindow = FixedWindowAggregation.windowNumber(shift, windowSize, last); + long sum = 0; + for (long wn = firstWindow; wn <= lastWindow; wn++) { + final long ws = FixedWindowAggregation.windowStartForWindowNumber(shift, windowSize, wn); + sum += FixedWindowAggregation.bytesInWindow(delta, last, v, ws, ws + windowSize - 1); + } + assertEquals("windowSize=" + windowSize + " shift=" + shift + " delta=" + delta + " last=" + last + " v=" + v, + whole, sum); + } + } + + /** With exact integer arithmetic (integer multiplier), the whole-flow total is the floored scaled byte count. */ + @Test + public void wholeFlowTotalEqualsFlooredScaledBytes() { + final Random rnd = new Random(99L); + for (int i = 0; i < 100000; i++) { + final long delta = 1_000_000_000L + (long) rnd.nextInt(1_000_000); + final long last = delta + (long) rnd.nextInt(100000); + final long bytes = 1 + rnd.nextInt(1_000_000); + final long mult = 1 + rnd.nextInt(1000); // integer multiplier -> V is an exact long + final double v = (double) bytes * mult; + assertEquals(bytes * mult, FixedWindowAggregation.bytesInWindow(delta, last, v, delta, last)); + } + } + + // ---- fidelity to the ideal real-valued proration ------------------------------------------ + + /** Each window's integer share is within one byte of {@code V * overlapMs / durationMs}. */ + @Test + public void perWindowShareTracksIdealProration() { + final Random rnd = new Random(7L); + for (int i = 0; i < 200000; i++) { + final long windowSize = 1 + (long) rnd.nextInt(50000); + final long delta = 1_000_000_000L + (long) rnd.nextInt(1_000_000); + final long last = delta + (long) rnd.nextInt(500000); + final double v = (1 + rnd.nextInt(1_000_000)) * randomSampling(rnd); + final long durationMs = last - delta + 1; + + final long firstWindow = FixedWindowAggregation.windowNumber(0L, windowSize, delta); + final long lastWindow = FixedWindowAggregation.windowNumber(0L, windowSize, last); + for (long wn = firstWindow; wn <= lastWindow; wn++) { + final long ws = FixedWindowAggregation.windowStartForWindowNumber(0L, windowSize, wn); + final long weInclusive = ws + windowSize - 1; + final long actual = FixedWindowAggregation.bytesInWindow(delta, last, v, ws, weInclusive); + final long overlapMs = Math.min(last, weInclusive) - Math.max(delta, ws) + 1; + final double ideal = v * overlapMs / durationMs; + assertTrue("actual=" + actual + " ideal=" + ideal + " windowSize=" + windowSize, + Math.abs(actual - ideal) < 1.0 + 1.0E-6); + } + } + } + + // ---- windowing invariants (mirror Nephron's UnalignedFixedWindowsTest) --------------------- + + @Test + public void shiftedWindowContainsTimestampAndRoundTrips() { + final Random rnd = new Random(1L); + for (int i = 0; i < 200000; i++) { + final int nodeId = rnd.nextInt(100000); + final long windowSize = 1 + (long) rnd.nextInt(100000); + final long shift = FixedWindowAggregation.perNodeShift(nodeId, windowSize); + final long timestamp = shift + (long) rnd.nextInt(10_000_000); // precondition: timestamp >= shift + + final long start = FixedWindowAggregation.windowStartForTimestamp(shift, windowSize, timestamp); + // unshifted start is a multiple of the window size, and the window contains the timestamp + assertEquals(0L, (start - shift) % windowSize); + assertTrue(start <= timestamp && start + windowSize > timestamp); + + // windowStartForTimestamp and windowNumber -> windowStartForWindowNumber are consistent + final long wn = FixedWindowAggregation.windowNumber(shift, windowSize, timestamp); + assertEquals(start, FixedWindowAggregation.windowStartForWindowNumber(shift, windowSize, wn)); + } + } + + @Test + public void globalGridWindowContainsTimestampAndRoundTrips() { + final Random rnd = new Random(2L); + for (int i = 0; i < 200000; i++) { + final long windowSize = 1 + (long) rnd.nextInt(100000); + final long timestamp = (long) (rnd.nextDouble() * 1_000_000_000_000L); + final long start = FixedWindowAggregation.windowStartForTimestamp(0L, windowSize, timestamp); + assertEquals(0L, start % windowSize); + assertTrue(start <= timestamp && start + windowSize > timestamp); + final long wn = FixedWindowAggregation.windowNumber(0L, windowSize, timestamp); + assertEquals(start, FixedWindowAggregation.windowStartForWindowNumber(0L, windowSize, wn)); + } + } + + // ---- lock the inlined fastutil hash -------------------------------------------------------- + + /** Golden values computed from fastutil 8.5.2 HashCommon.mix (the version Nephron pins), windowSize = 60000. */ + @Test + public void perNodeShiftMatchesFastutilGoldenValues() { + final long ws = 60000L; + final long[] expected = {0L, 43410L, 14940L, 28243L, 29881L, 42824L, 2022L, 41611L, 47533L, 48846L, 25648L}; + for (int nodeId = 0; nodeId <= 10; nodeId++) { + assertEquals("perNodeShift(" + nodeId + ", " + ws + ")", expected[nodeId], + FixedWindowAggregation.perNodeShift(nodeId, ws)); + } + } + + /** + * An effective multiplier, i.e. what {@link FixedWindowAggregation#samplingMultiplier} would + * produce for a variety of raw sampling intervals (degenerate values collapse to 1.0). Always finite + * and positive, so a flow's scaled byte total is well defined. + */ + private static double randomSampling(final Random rnd) { + switch (rnd.nextInt(5)) { + case 0: return FixedWindowAggregation.samplingMultiplier(1.0); + case 1: return FixedWindowAggregation.samplingMultiplier(0.0); // -> 1.0 + case 2: return FixedWindowAggregation.samplingMultiplier(Double.NaN); // -> 1.0 + case 3: return FixedWindowAggregation.samplingMultiplier(1.0 + rnd.nextInt(1000)); + default: return FixedWindowAggregation.samplingMultiplier((double) (1 + rnd.nextInt(1000))); + } + } +} diff --git a/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/FlowAggIT.java b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/FlowAggIT.java new file mode 100644 index 000000000000..b02e56dbef56 --- /dev/null +++ b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/FlowAggIT.java @@ -0,0 +1,285 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.postgres; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.sql.Connection; + +import org.junit.Assume; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.postgresql.ds.PGSimpleDataSource; +import org.springframework.jdbc.core.JdbcTemplate; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.PostgreSQLContainer; + +import com.codahale.metrics.MetricRegistry; + +import liquibase.Liquibase; +import liquibase.database.DatabaseFactory; +import liquibase.database.jvm.JdbcConnection; +import liquibase.resource.ClassLoaderResourceAccessor; + +/** + * End-to-end verification of the write-time aggregator against a real PostgreSQL: {@link FlowAggregator} + * feeding {@link FlowAggWriter} into the {@code flow_agg} table, then reading the rows back. Also proves + * the distribution model — several writers each emit partial rows per {@code (window, key)} and the + * reader reconstructs the total by summing them. + */ +public class FlowAggIT { + + private static final String CHANGELOG = "org/opennms/netmgt/flows/postgres/changelog.xml"; + // a window boundary on the global (shift 0) 1000 ms grid + private static final long BASE = 1700000000000L; + private static final long WINDOW = 1000L; + + private static PostgreSQLContainer POSTGRES; + private static PGSimpleDataSource DATA_SOURCE; + private static JdbcTemplate jdbc; + + @BeforeClass + public static void startContainer() throws Exception { + final String externalUrl = System.getProperty("it.postgres.jdbcUrl"); + final String url; + final String user; + final String password; + if (externalUrl != null && !externalUrl.isEmpty()) { + url = externalUrl; + user = System.getProperty("it.postgres.user", "postgres"); + password = System.getProperty("it.postgres.password", "postgres"); + } else { + Assume.assumeTrue("No Docker/podman runtime available; skipping live-SQL IT.", + DockerClientFactory.instance().isDockerAvailable()); + POSTGRES = new PostgreSQLContainer<>("postgres:15"); + POSTGRES.start(); + url = POSTGRES.getJdbcUrl(); + user = POSTGRES.getUsername(); + password = POSTGRES.getPassword(); + } + DATA_SOURCE = new PGSimpleDataSource(); + DATA_SOURCE.setUrl(url); + DATA_SOURCE.setUser(user); + DATA_SOURCE.setPassword(password); + DATA_SOURCE.setReWriteBatchedInserts(true); + jdbc = new JdbcTemplate(DATA_SOURCE); + try (Connection conn = DATA_SOURCE.getConnection()) { + final liquibase.database.Database db = DatabaseFactory.getInstance() + .findCorrectDatabaseImplementation(new JdbcConnection(conn)); + new Liquibase(CHANGELOG, new ClassLoaderResourceAccessor(FlowAggIT.class.getClassLoader()), db).update(""); + } + } + + @Before + public void truncate() { + jdbc.execute("TRUNCATE flow_agg"); + } + + private static FlowInput flow(final int exporter, final int ifIndex, final long delta, final long last, + final long bytes, final boolean ingress, final String app, final String convo, + final String src, final String dst, final Integer ecn) { + return new FlowInput(delta, last, bytes, null, ingress, exporter, ifIndex, app, convo, src, dst, + src == null ? null : src + "-host", dst == null ? null : dst + "-host", ecn, null); + } + + private static FlowAggregator aggregatorWriting(final String writerId) { + return new FlowAggregator(WINDOW, 0L, 3_600_000L, 10, 0L, + new FlowAggWriter(DATA_SOURCE, writerId), new MetricRegistry()); + } + + private long sumTotal(final int exporter, final String dimension, final String key) { + final Long v; + if (key == null) { + v = jdbc.queryForObject("SELECT COALESCE(SUM(bytes_in + bytes_out), 0) FROM flow_agg " + + "WHERE exporter_node_id = ? AND dimension = ? AND grouped_by_key IS NULL", Long.class, exporter, dimension); + } else { + v = jdbc.queryForObject("SELECT COALESCE(SUM(bytes_in + bytes_out), 0) FROM flow_agg " + + "WHERE exporter_node_id = ? AND dimension = ? AND grouped_by_key = ?", Long.class, exporter, dimension, key); + } + return v != null ? v : 0L; + } + + @Test + public void aggregatesPersistAndReadSumReconstructsTotals() { + final int exporter = 42; + // 3000 bytes uniformly over three 1000 ms windows [BASE, BASE+2999] + try (FlowAggregator agg = aggregatorWriting("core")) { + agg.add(flow(exporter, 5, BASE, BASE + 2999, 3000L, true, "http", "conv1", "10.0.0.1", "10.0.0.2", 0)); + } // close() flushes all windows synchronously + + // one INTERFACE row per spanned window + final int interfaceRows = jdbc.queryForObject( + "SELECT COUNT(*) FROM flow_agg WHERE exporter_node_id = ? AND dimension = 'INTERFACE'", Integer.class, exporter); + assertEquals(3, interfaceRows); + + // read-time SUM over windows reconstructs the flow's total on every dimension (conservation through the DB) + assertEquals(3000L, sumTotal(exporter, "INTERFACE", null)); + assertEquals(3000L, sumTotal(exporter, "APPLICATION", "http")); + assertEquals(3000L, sumTotal(exporter, "CONVERSATION", "conv1")); + assertEquals(3000L, sumTotal(exporter, "HOST", "10.0.0.1")); + assertEquals(3000L, sumTotal(exporter, "HOST", "10.0.0.2")); + + // ingress -> bytes_in; ecn 0 -> non-ECT + assertEquals(Long.valueOf(3000L), jdbc.queryForObject( + "SELECT SUM(bytes_in) FROM flow_agg WHERE exporter_node_id = ? AND dimension = 'INTERFACE'", Long.class, exporter)); + assertEquals(Integer.valueOf(0), jdbc.queryForObject( + "SELECT COUNT(*) FROM flow_agg WHERE exporter_node_id = ? AND non_ecn_capable_transport = false", Integer.class, exporter)); + } + + @Test + public void tosDimensionPersistsWithAndWithoutTosRows() { + final int exporter = 88; + try (FlowAggregator agg = aggregatorWriting("core")) { + // one flow with DSCP 5 -> a without-TOS (dscp NULL) row AND a with-TOS (dscp 5) row, each 1000 bytes + agg.add(new FlowInput(BASE, BASE + 999, 1000L, null, true, exporter, 1, "http", null, null, null, + null, null, null, 5)); + } + assertEquals("without-TOS rollup", Long.valueOf(1000L), jdbc.queryForObject( + "SELECT sum(bytes_in+bytes_out) FROM flow_agg WHERE exporter_node_id=? AND dimension='INTERFACE' AND dscp IS NULL", + Long.class, exporter)); + assertEquals("with-TOS dscp=5", Long.valueOf(1000L), jdbc.queryForObject( + "SELECT sum(bytes_in+bytes_out) FROM flow_agg WHERE exporter_node_id=? AND dimension='INTERFACE' AND dscp=5", + Long.class, exporter)); + } + + @Test + public void aggregatedReadServiceServesTopNApplicationSummaries() throws Exception { + final int exporter = 55; + try (FlowAggregator agg = aggregatorWriting("core")) { + // four applications in one window, distinct byte totals (ingress) + agg.add(flow(exporter, 1, BASE, BASE + 999, 400L, true, "a1", null, null, null, null)); + agg.add(flow(exporter, 1, BASE, BASE + 999, 300L, true, "a2", null, null, null, null)); + agg.add(flow(exporter, 1, BASE, BASE + 999, 200L, true, "a3", null, null, null, null)); + agg.add(flow(exporter, 1, BASE, BASE + 999, 100L, true, "a4", null, null, null, null)); + } + + final AggregatedFlowQueryService qs = + new AggregatedFlowQueryService(org.mockito.Mockito.mock(org.opennms.netmgt.flows.api.FlowQueryService.class)); + qs.setDataSource(DATA_SOURCE); + qs.start(); + try { + final java.util.List filters = + java.util.Collections.singletonList(new org.opennms.netmgt.flows.filter.api.TimeRangeFilter(BASE, BASE + WINDOW)); + // top 2 applications + Other + final java.util.List> out = + qs.getTopNApplicationSummaries(2, true, filters).get(10, java.util.concurrent.TimeUnit.SECONDS); + + assertEquals("top 2 + Other", 3, out.size()); + assertEquals("a1", out.get(0).getEntity()); + assertEquals(400L, out.get(0).getBytesIn()); + assertEquals("a2", out.get(1).getEntity()); + assertEquals(300L, out.get(1).getBytesIn()); + assertEquals("Other", out.get(2).getEntity()); + assertEquals("Other = a3 + a4", 300L, out.get(2).getBytesIn()); + final long total = out.stream().mapToLong(org.opennms.netmgt.flows.api.TrafficSummary::getBytesIn).sum(); + assertEquals("top-N + Other reconstruct the dimension total", 1000L, total); + } finally { + qs.stop(); + } + } + + @Test + public void aggregatedReadServiceServesTopNApplicationSeries() throws Exception { + final int exporter = 66; + try (FlowAggregator agg = aggregatorWriting("core")) { + // Feed in event-time order (allowedLateness=0): both window-1 flows, then the window-2 flow. + // a1 spans both windows; a2 is only in window 1. + agg.add(flow(exporter, 1, BASE, BASE + 999, 300L, true, "a1", null, null, null, null)); + agg.add(flow(exporter, 1, BASE, BASE + 999, 100L, true, "a2", null, null, null, null)); + agg.add(flow(exporter, 1, BASE + 1000, BASE + 1999, 400L, true, "a1", null, null, null, null)); + } + final AggregatedFlowQueryService qs = + new AggregatedFlowQueryService(org.mockito.Mockito.mock(org.opennms.netmgt.flows.api.FlowQueryService.class)); + qs.setDataSource(DATA_SOURCE); + qs.start(); + try { + final java.util.List filters = + java.util.Collections.singletonList(new org.opennms.netmgt.flows.filter.api.TimeRangeFilter(BASE, BASE + 2 * WINDOW)); + // top-1 app (a1) + Other, one bucket per WINDOW-ms step (== aggregation window here) + final com.google.common.collect.Table, Long, Double> t = + qs.getTopNApplicationSeries(1, WINDOW, true, filters).get(10, java.util.concurrent.TimeUnit.SECONDS); + + final org.opennms.netmgt.flows.api.Directional a1In = new org.opennms.netmgt.flows.api.Directional<>("a1", true); + final org.opennms.netmgt.flows.api.Directional otherIn = new org.opennms.netmgt.flows.api.Directional<>("Other", true); + assertEquals(300.0, t.get(a1In, BASE), 0.0); // a1, window 1 + assertEquals(400.0, t.get(a1In, BASE + WINDOW), 0.0); // a1, window 2 + assertEquals("Other = a2 in window 1", 100.0, t.get(otherIn, BASE), 0.0); + assertNull("no Other in window 2 (only a1 there)", t.get(otherIn, BASE + WINDOW)); + assertNull("a2 folded into Other, not its own row", t.get(new org.opennms.netmgt.flows.api.Directional<>("a2", true), BASE)); + } finally { + qs.stop(); + } + } + + @Test + public void aggregatedReadServiceServesDscpFieldValuesAndSummaries() throws Exception { + final int exporter = 91; + try (FlowAggregator agg = aggregatorWriting("core")) { + // two DSCP classes in one window (with-TOS INTERFACE rows) + agg.add(new FlowInput(BASE, BASE + 999, 300L, null, true, exporter, 1, "a1", null, null, null, null, null, null, 0)); + agg.add(new FlowInput(BASE, BASE + 999, 700L, null, true, exporter, 1, "a2", null, null, null, null, null, null, 46)); + } + final AggregatedFlowQueryService qs = + new AggregatedFlowQueryService(org.mockito.Mockito.mock(org.opennms.netmgt.flows.api.FlowQueryService.class)); + qs.setDataSource(DATA_SOURCE); + qs.start(); + try { + final java.util.List filters = + java.util.Collections.singletonList(new org.opennms.netmgt.flows.filter.api.TimeRangeFilter(BASE, BASE + WINDOW)); + + final java.util.List values = + qs.getFieldValues(org.opennms.netmgt.flows.api.LimitedCardinalityField.DSCP, filters).get(10, java.util.concurrent.TimeUnit.SECONDS); + assertEquals(java.util.Arrays.asList("0", "46"), values); + + final java.util.List> sums = + qs.getFieldSummaries(org.opennms.netmgt.flows.api.LimitedCardinalityField.DSCP, filters).get(10, java.util.concurrent.TimeUnit.SECONDS); + assertEquals(2, sums.size()); + assertEquals("46", sums.get(0).getEntity()); // largest first + assertEquals(700L, sums.get(0).getBytesIn()); + assertEquals("0", sums.get(1).getEntity()); + assertEquals(300L, sums.get(1).getBytesIn()); + } finally { + qs.stop(); + } + } + + @Test + public void partialRowsFromMultipleWritersSumOnRead() { + final int exporter = 77; + // Two independent writers (e.g. two Sentinels) each see the SAME flow and emit their own partials. + for (final String writerId : new String[] {"sentinel-a", "sentinel-b"}) { + try (FlowAggregator agg = aggregatorWriting(writerId)) { + agg.add(flow(exporter, 1, BASE, BASE + 999, 1000L, true, "https", null, null, null, null)); + } + } + // two partial rows for the single (window, interface) key... + assertEquals(Integer.valueOf(2), jdbc.queryForObject( + "SELECT COUNT(*) FROM flow_agg WHERE exporter_node_id = ? AND dimension = 'INTERFACE'", Integer.class, exporter)); + assertEquals(Integer.valueOf(2), jdbc.queryForObject( + "SELECT COUNT(DISTINCT writer_id) FROM flow_agg WHERE exporter_node_id = ?", Integer.class, exporter)); + // ...and the reader reconstructs the combined total by summing the partials + assertEquals(2000L, sumTotal(exporter, "INTERFACE", null)); + assertEquals(2000L, sumTotal(exporter, "APPLICATION", "https")); + } +} diff --git a/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/FlowAggregatorTest.java b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/FlowAggregatorTest.java new file mode 100644 index 000000000000..493d05e0dfd9 --- /dev/null +++ b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/FlowAggregatorTest.java @@ -0,0 +1,338 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.postgres; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.function.Consumer; + +import org.opennms.integration.api.v1.flows.Flow; + +import org.junit.Test; + +import com.codahale.metrics.MetricRegistry; + +/** + * Exercises the {@link FlowAggregator} engine in isolation (no database): windowing, proration fan-out + * across dimensions, watermark-driven close, conservation, direction/ECN handling, and late-flow drop. + * The background flush thread is intentionally not started; {@link FlowAggregator#flushClosedWindows()} + * is driven directly so the tests are deterministic. + */ +public class FlowAggregatorTest { + + private static final long SIZE = 1000L; + + private final List captured = new ArrayList<>(); + private final Consumer> sink = captured::addAll; + + private FlowAggregator aggregator(final long latenessMs) { + return aggregator(latenessMs, 0); // uncapped + } + + private FlowAggregator aggregator(final long latenessMs, final int topK) { + // huge flush interval: we drive flushes manually, the thread is never started + return new FlowAggregator(SIZE, latenessMs, 3_600_000L, topK, 0L, sink, new MetricRegistry()); + } + + private static FlowInput flow(final long delta, final long last, final long bytes, final Double sampling, + final boolean ingress, final int exporter, final int ifIndex, final String app, + final String convo, final String src, final String dst, final Integer ecn) { + return flowWithDscp(delta, last, bytes, sampling, ingress, exporter, ifIndex, app, convo, src, dst, ecn, null); + } + + private static FlowInput flowWithDscp(final long delta, final long last, final long bytes, final Double sampling, + final boolean ingress, final int exporter, final int ifIndex, final String app, + final String convo, final String src, final String dst, final Integer ecn, + final Integer dscp) { + return new FlowInput(delta, last, bytes, sampling, ingress, exporter, ifIndex, app, convo, src, dst, + src == null ? null : src + "-host", dst == null ? null : dst + "-host", ecn, dscp); + } + + /** Push the watermark to {@code ts} with a throwaway flow on an unrelated exporter (its window stays open). */ + private void advanceWatermark(final FlowAggregator agg, final long ts) { + agg.add(flow(ts, ts, 1L, null, true, 999_999, 1, null, null, null, null, null)); + } + + /** Find a without-TOS (dscp == null) row. */ + private AggregatedFlow row(final long windowStart, final AggregatedFlow.Dimension dim, final String key, final int exporter) { + return rowTos(windowStart, dim, key, exporter, null); + } + + /** Find a row for a specific DSCP scope (dscp == null selects the without-TOS rollup). */ + private AggregatedFlow rowTos(final long windowStart, final AggregatedFlow.Dimension dim, final String key, + final int exporter, final Integer dscp) { + return captured.stream() + .filter(r -> r.windowStartMs == windowStart && r.dimension == dim + && Objects.equals(r.groupedByKey, key) && r.exporterNodeId == exporter + && Objects.equals(r.dscp, dscp)) + .findFirst().orElse(null); + } + + @Test + public void singleFlowFansOutToInterfaceAndDimensionRows() { + final FlowAggregator agg = aggregator(0L); + // flow fully inside [1000, 2000): 1000 ingress bytes, no sampling + agg.add(flow(1000L, 1999L, 1000L, null, true, 1, 10, "http", "c1", "10.0.0.1", "10.0.0.2", null)); + advanceWatermark(agg, 10_000L); // closes [1000,2000) + agg.flushClosedWindows(); + + final AggregatedFlow itf = row(1000L, AggregatedFlow.Dimension.INTERFACE, null, 1); + assertNotNull(itf); + assertEquals(1000L, itf.bytesIn); + assertEquals(0L, itf.bytesOut); + assertEquals(2000L, itf.windowEndMs); + assertEquals(10, itf.ifIndex); + assertTrue("ecn absent -> non-ECT", itf.nonEcnCapableTransport); + assertFalse(itf.congestionEncountered); + + assertEquals(1000L, row(1000L, AggregatedFlow.Dimension.APPLICATION, "http", 1).bytesIn); + assertEquals(1000L, row(1000L, AggregatedFlow.Dimension.CONVERSATION, "c1", 1).bytesIn); + final AggregatedFlow srcHost = row(1000L, AggregatedFlow.Dimension.HOST, "10.0.0.1", 1); + assertEquals(1000L, srcHost.bytesIn); + assertEquals("10.0.0.1-host", srcHost.hostname); + assertEquals(1000L, row(1000L, AggregatedFlow.Dimension.HOST, "10.0.0.2", 1).bytesIn); + } + + @Test + public void flowSpanningTwoWindowsSplitsAndConserves() { + final FlowAggregator agg = aggregator(0L); + // 2000 bytes uniformly over [1000, 2999] -> ~1000 per 1000ms window + agg.add(flow(1000L, 2999L, 2000L, null, true, 1, 10, "http", null, null, null, null)); + advanceWatermark(agg, 10_000L); + agg.flushClosedWindows(); + + final AggregatedFlow w1 = row(1000L, AggregatedFlow.Dimension.INTERFACE, null, 1); + final AggregatedFlow w2 = row(2000L, AggregatedFlow.Dimension.INTERFACE, null, 1); + assertNotNull(w1); + assertNotNull(w2); + assertEquals(1000L, w1.bytesIn); + assertEquals(1000L, w2.bytesIn); + assertEquals("conservation across windows", 2000L, w1.bytesIn + w2.bytesIn); + } + + @Test + public void egressAndEcnFlagsAreAggregated() { + final FlowAggregator agg = aggregator(0L); + agg.add(flow(1000L, 1999L, 500L, null, false, 7, 3, "https", null, null, null, 3)); // egress, CE + advanceWatermark(agg, 10_000L); + agg.flushClosedWindows(); + + final AggregatedFlow itf = row(1000L, AggregatedFlow.Dimension.INTERFACE, null, 7); + assertNotNull(itf); + assertEquals(0L, itf.bytesIn); + assertEquals(500L, itf.bytesOut); + assertTrue("ecn == 3 -> congestion encountered", itf.congestionEncountered); + assertFalse("ecn == 3 is not non-ECT", itf.nonEcnCapableTransport); + } + + @Test + public void windowsDoNotFlushUntilClosed() { + final FlowAggregator agg = aggregator(0L); + agg.add(flow(1000L, 1999L, 1000L, null, true, 1, 10, null, null, null, null, null)); + // watermark == 1999 < 2000, so [1000,2000) is still open + agg.flushClosedWindows(); + assertTrue("nothing should be flushed yet", captured.isEmpty()); + + advanceWatermark(agg, 2000L); // watermark 2000 -> threshold 1000 -> [1000,2000) closes + agg.flushClosedWindows(); + assertNotNull(row(1000L, AggregatedFlow.Dimension.INTERFACE, null, 1)); + } + + @Test + public void lateFlowForAlreadyClosedWindowIsDropped() { + final FlowAggregator agg = aggregator(0L); + advanceWatermark(agg, 10_000L); // watermark high; [1000,2000) is already closed + final boolean accepted = agg.add(flow(1000L, 1500L, 1000L, null, true, 1, 10, "http", null, null, null, null)); + assertFalse("a flow for a closed window must be rejected", accepted); + agg.flushClosedWindows(); + // no rows for the closed window / exporter 1 + assertTrue(captured.stream().noneMatch(r -> r.exporterNodeId == 1)); + } + + @Test + public void allowedLatenessHoldsWindowOpenForLateData() { + final FlowAggregator agg = aggregator(5000L); // 5s lateness + agg.add(flow(1000L, 1999L, 1000L, null, true, 1, 10, "http", null, null, null, null)); + advanceWatermark(agg, 2500L); // watermark 2500: without lateness [1000,2000) would close, but 2500-1000-5000<1000 + agg.flushClosedWindows(); + assertTrue("lateness should keep the window open", captured.isEmpty()); + + // a late flow for the still-open window is accepted and merged + assertTrue(agg.add(flow(1500L, 1600L, 500L, null, true, 1, 10, "http", null, null, null, null))); + advanceWatermark(agg, 6001L); // now 6001-1000-5000 = 1 >= 1000? no... push further + advanceWatermark(agg, 7000L); // 7000-1000-5000 = 1000 >= 1000 -> closes [1000,2000) + agg.flushClosedWindows(); + final AggregatedFlow itf = row(1000L, AggregatedFlow.Dimension.INTERFACE, null, 1); + assertNotNull(itf); + assertEquals("both the original and the late flow are included", 1500L, itf.bytesIn); + } + + @Test + public void closeFlushesRemainingOpenWindows() { + final FlowAggregator agg = aggregator(0L); + agg.add(flow(1000L, 1999L, 1000L, null, true, 1, 10, "http", null, null, null, null)); + // window is still open (never advanced the watermark past it) + agg.close(); + assertNotNull("close() must flush buffered windows", row(1000L, AggregatedFlow.Dimension.INTERFACE, null, 1)); + } + + @Test + public void interfaceTotalConservesAcrossManyWindows() { + final FlowAggregator agg = new FlowAggregator(100L, 0L, 3_600_000L, 0, 0L, sink, new MetricRegistry()); + final long bytes = 123_457L; + agg.add(flow(100_000L, 137_777L, bytes, null, true, 1, 10, null, null, null, null, null)); // spans many 100ms windows + agg.close(); // flush everything + + final long sum = captured.stream() + .filter(r -> r.dimension == AggregatedFlow.Dimension.INTERFACE && r.exporterNodeId == 1) + .mapToLong(r -> r.bytesIn) + .sum(); + assertEquals("sum of per-window interface bytes equals the flow's bytes", bytes, sum); + } + + @Test + public void topKCapsCappedDimensionsAndRollsRemainderIntoOther() { + final FlowAggregator agg = aggregator(0L, 2); // keep top 2 conversations per interface + // five conversations on one exporter/interface, distinct byte counts, all in [1000,2000) + final long[] bytes = {500, 400, 300, 200, 100}; + for (int i = 0; i < bytes.length; i++) { + agg.add(flow(1000L, 1999L, bytes[i], null, true, 1, 10, null, "c" + i, null, null, null)); + } + advanceWatermark(agg, 10_000L); + agg.flushClosedWindows(); + + // top 2 by bytes are c0 (500) and c1 (400), stored individually + assertEquals(500L, row(1000L, AggregatedFlow.Dimension.CONVERSATION, "c0", 1).bytesIn); + assertEquals(400L, row(1000L, AggregatedFlow.Dimension.CONVERSATION, "c1", 1).bytesIn); + // c2..c4 are not stored individually... + assertNull(row(1000L, AggregatedFlow.Dimension.CONVERSATION, "c2", 1)); + // ...they roll into one null-key "Other" row: 300 + 200 + 100 + assertEquals(600L, row(1000L, AggregatedFlow.Dimension.CONVERSATION, null, 1).bytesIn); + // exactly three conversation rows: 2 top-K + 1 Other + assertEquals(3L, captured.stream() + .filter(r -> r.dimension == AggregatedFlow.Dimension.CONVERSATION && r.exporterNodeId == 1).count()); + // conservation: capped rows + Other still sum to the (uncapped) interface total + final long convTotal = captured.stream() + .filter(r -> r.dimension == AggregatedFlow.Dimension.CONVERSATION && r.exporterNodeId == 1) + .mapToLong(r -> r.bytesIn).sum(); + assertEquals(1500L, convTotal); + assertEquals(1500L, row(1000L, AggregatedFlow.Dimension.INTERFACE, null, 1).bytesIn); + } + + @Test + public void interfaceTotalsAreNeverCappedByTopK() { + final FlowAggregator agg = aggregator(0L, 1); // aggressive cap + // three interfaces on one exporter, each with a single conversation + agg.add(flow(1000L, 1999L, 100L, null, true, 1, 10, null, "a", null, null, null)); + agg.add(flow(1000L, 1999L, 200L, null, true, 1, 11, null, "b", null, null, null)); + agg.add(flow(1000L, 1999L, 300L, null, true, 1, 12, null, "c", null, null, null)); + advanceWatermark(agg, 10_000L); + agg.flushClosedWindows(); + + // all three interface totals are present despite topK=1 + assertEquals(3L, captured.stream() + .filter(r -> r.dimension == AggregatedFlow.Dimension.INTERFACE && r.exporterNodeId == 1).count()); + } + + @Test + public void tosProducesWithAndWithoutTosAggregationsThatReconcile() { + final FlowAggregator agg = aggregator(0L, 10); + // same exporter/interface/app/conversation, two different DSCP values, in [1000,2000) + agg.add(flowWithDscp(1000L, 1999L, 300L, null, true, 1, 10, "http", "c1", null, null, null, 0)); + agg.add(flowWithDscp(1000L, 1999L, 700L, null, true, 1, 10, "http", "c1", null, null, null, 46)); + advanceWatermark(agg, 10_000L); + agg.flushClosedWindows(); + + // without-TOS rollup (dscp == null): both DSCP merged + assertEquals(1000L, row(1000L, AggregatedFlow.Dimension.INTERFACE, null, 1).bytesIn); + assertEquals(1000L, row(1000L, AggregatedFlow.Dimension.APPLICATION, "http", 1).bytesIn); + assertEquals(1000L, row(1000L, AggregatedFlow.Dimension.CONVERSATION, "c1", 1).bytesIn); + + // with-TOS: separate per-DSCP aggregations + assertEquals(300L, rowTos(1000L, AggregatedFlow.Dimension.INTERFACE, null, 1, 0).bytesIn); + assertEquals(700L, rowTos(1000L, AggregatedFlow.Dimension.INTERFACE, null, 1, 46).bytesIn); + assertEquals(300L, rowTos(1000L, AggregatedFlow.Dimension.APPLICATION, "http", 1, 0).bytesIn); + assertEquals(700L, rowTos(1000L, AggregatedFlow.Dimension.APPLICATION, "http", 1, 46).bytesIn); + + // conservation: the per-DSCP interface totals sum to the without-TOS total + final long withTosSum = rowTos(1000L, AggregatedFlow.Dimension.INTERFACE, null, 1, 0).bytesIn + + rowTos(1000L, AggregatedFlow.Dimension.INTERFACE, null, 1, 46).bytesIn; + assertEquals(row(1000L, AggregatedFlow.Dimension.INTERFACE, null, 1).bytesIn, withTosSum); + } + + @Test + public void topKIsAppliedPerDscpScope() { + final FlowAggregator agg = aggregator(0L, 1); // keep top 1 per (interface, dscp) + // dscp 0: c1 (500) beats c2 (100); dscp 46: c3 (300) only + agg.add(flowWithDscp(1000L, 1999L, 500L, null, true, 1, 10, null, "c1", null, null, null, 0)); + agg.add(flowWithDscp(1000L, 1999L, 100L, null, true, 1, 10, null, "c2", null, null, null, 0)); + agg.add(flowWithDscp(1000L, 1999L, 300L, null, true, 1, 10, null, "c3", null, null, null, 46)); + advanceWatermark(agg, 10_000L); + agg.flushClosedWindows(); + + // dscp 0: c1 is the single top-K, c2 falls into Other (null key) + assertEquals(500L, rowTos(1000L, AggregatedFlow.Dimension.CONVERSATION, "c1", 1, 0).bytesIn); + assertNull(rowTos(1000L, AggregatedFlow.Dimension.CONVERSATION, "c2", 1, 0)); + assertEquals(100L, rowTos(1000L, AggregatedFlow.Dimension.CONVERSATION, null, 1, 0).bytesIn); + // dscp 46: c3 alone, no Other + assertEquals(300L, rowTos(1000L, AggregatedFlow.Dimension.CONVERSATION, "c3", 1, 46).bytesIn); + assertNull(rowTos(1000L, AggregatedFlow.Dimension.CONVERSATION, null, 1, 46)); + } + + @Test + public void idleFlushReleasesWindowsWhenTheWatermarkStalls() { + final long[] now = {1_000_000L}; + // idleFlushMs = 5000; drive the wall clock manually + final FlowAggregator agg = new FlowAggregator(SIZE, 0L, 3_600_000L, 0, 5000L, sink, new MetricRegistry()); + agg.setClock(() -> now[0]); + + // one flow in [1000,2000): watermark = 1999, which does NOT close the window + agg.add(flow(1000L, 1999L, 1000L, null, true, 1, 10, "http", null, null, null, null)); + agg.flushClosedWindows(); + assertTrue("event-time close should not fire with a stalled watermark", captured.isEmpty()); + + // not old enough yet (4s < 5s) + now[0] += 4000L; + agg.flushIdleWindows(); + assertTrue(captured.isEmpty()); + + // past the idle timeout (6s > 5s): the window is released despite the stalled watermark + now[0] += 2000L; + agg.flushIdleWindows(); + assertNotNull(row(1000L, AggregatedFlow.Dimension.INTERFACE, null, 1)); + assertEquals(1000L, row(1000L, AggregatedFlow.Dimension.INTERFACE, null, 1).bytesIn); + } + + @Test + public void fromFlowSkipsFlowsMissingRequiredFields() { + // a Flow with no switched timestamps / exporter yields no FlowInput (Mockito returns null/empty) + assertNull(FlowInput.from(mock(Flow.class))); + } +} diff --git a/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowDisabledTest.java b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowDisabledTest.java index cd66649888d2..6cfc04710a2d 100644 --- a/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowDisabledTest.java +++ b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresFlowDisabledTest.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.util.Collections; import java.util.List; @@ -33,6 +34,7 @@ import org.junit.Test; import org.opennms.core.health.api.Response; import org.opennms.core.health.api.Status; +import org.opennms.distributed.core.api.Identity; import org.opennms.integration.api.v1.flows.Flow; import org.opennms.netmgt.flows.filter.api.Filter; import org.opennms.netmgt.flows.filter.api.TimeRangeFilter; @@ -88,6 +90,44 @@ public void queryServiceIsInertWhenTheProviderHasNoDataSource() throws Exception queryService.stop(); } + @Test + public void aggregationWriterIdDefaultsToNodeIdentityWhenBlank() { + final AggregatingFlowRepository agg = new AggregatingFlowRepository(new MetricRegistry()); + final Identity id = mock(Identity.class); + when(id.getId()).thenReturn("sentinel-7"); + agg.setIdentity(id); + agg.setWriterId(""); // blank -> auto-default to the node identity + assertEquals("sentinel-7", agg.resolveWriterId()); + agg.setWriterId("explicit-id"); // an explicit value wins + assertEquals("explicit-id", agg.resolveWriterId()); + } + + @Test + public void aggregationWriterIdFallsBackToCoreWithoutIdentity() { + final AggregatingFlowRepository agg = new AggregatingFlowRepository(new MetricRegistry()); + agg.setWriterId(""); // blank, and no Identity injected + assertEquals("core", agg.resolveWriterId()); + } + + @Test + public void aggregatingRepositoryIsInertWhenDisabled() throws Exception { + final AggregatingFlowRepository agg = new AggregatingFlowRepository(new MetricRegistry()); + agg.setEnabled(false); // default; aggregation off + agg.start(); // must NOT throw or start a writer + agg.persist(Collections.singletonList(mock(Flow.class))); // must be a no-op + agg.stop(); + } + + @Test + public void aggregatingRepositoryIsInertWhenEnabledButNoDataSource() throws Exception { + final AggregatingFlowRepository agg = new AggregatingFlowRepository(new MetricRegistry()); + agg.setEnabled(true); + agg.setDataSourceProvider(inertProvider()); + agg.start(); // logs, but must NOT throw (feature loads) + agg.persist(Collections.singletonList(mock(Flow.class))); // must be a no-op, not an NPE + agg.stop(); + } + @Test public void healthCheckReportsNotConfiguredWhenNoDataSource() { // Mirrors the Elasticsearch flow health check: an installed-but-unconfigured backend is diff --git a/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresSmartQueryServiceTest.java b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresSmartQueryServiceTest.java new file mode 100644 index 000000000000..0f8a11d5c14e --- /dev/null +++ b/features/flows/postgres/src/test/java/org/opennms/netmgt/flows/postgres/PostgresSmartQueryServiceTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to The OpenNMS Group, Inc (TOG) under one or more + * contributor license agreements. See the LICENSE.md file + * distributed with this work for additional information + * regarding copyright ownership. + * + * TOG licenses this file to You under the GNU Affero General + * Public License Version 3 (the "License") or (at your option) + * any later version. You may not use this file except in + * compliance with the License. You may obtain a copy of the + * License at: + * + * https://www.gnu.org/licenses/agpl-3.0.txt + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the + * License. + */ +package org.opennms.netmgt.flows.postgres; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.junit.Before; +import org.junit.Test; +import org.opennms.netmgt.flows.api.FlowQueryService; +import org.opennms.netmgt.flows.filter.api.Filter; +import org.opennms.netmgt.flows.filter.api.TimeRangeFilter; + +import com.codahale.metrics.MetricRegistry; + +/** + * Verifies the raw-vs-aggregated routing of {@link PostgresSmartQueryService}, mirroring the + * Elasticsearch SmartQueryService semantics. + */ +public class PostgresSmartQueryServiceTest { + + private PostgresSmartQueryService svc; + + @Before + public void setUp() { + svc = new PostgresSmartQueryService(new MetricRegistry(), mock(FlowQueryService.class), mock(FlowQueryService.class)); + } + + private static List range(final long start, final long end) { + return Collections.singletonList(new TimeRangeFilter(start, end)); + } + + @Test + public void defaultsToRawForEverything() { + final long now = System.currentTimeMillis(); + assertEquals(PostgresSmartQueryService.QueryServiceType.RAW, svc.getDelegate(range(now - 3_600_000L, now), false)); + assertEquals(PostgresSmartQueryService.QueryServiceType.RAW, svc.getDelegate(range(now - 3_600_000L, now), true)); + } + + @Test + public void alwaysUseAggForcesAggEvenForSpecificEntities() { + svc.setAlwaysUseAggForQueries(true); + final long now = System.currentTimeMillis(); + assertEquals(PostgresSmartQueryService.QueryServiceType.AGG, svc.getDelegate(range(now - 60_000L, now), false)); + assertEquals(PostgresSmartQueryService.QueryServiceType.AGG, svc.getDelegate(range(now - 60_000L, now), true)); + } + + @Test + public void thresholdRoutingWhenNeitherFlagIsSet() { + svc.setAlwaysUseRawForQueries(false); // both flags now false -> threshold routing + final long now = System.currentTimeMillis(); + final long twoMin = TimeUnit.MINUTES.toMillis(2); + + // specific-entity queries always go raw + assertEquals(PostgresSmartQueryService.QueryServiceType.RAW, svc.getDelegate(range(now - 3 * twoMin, now), true)); + // a wide-enough duration goes aggregated + assertEquals(PostgresSmartQueryService.QueryServiceType.AGG, svc.getDelegate(range(now - 3 * twoMin, now), false)); + // a narrow, recent range stays raw + assertEquals(PostgresSmartQueryService.QueryServiceType.RAW, svc.getDelegate(range(now - 60_000L, now), false)); + // a narrow range whose endpoint is older than 7 days goes aggregated + final long old = now - TimeUnit.DAYS.toMillis(8); + assertEquals(PostgresSmartQueryService.QueryServiceType.AGG, svc.getDelegate(range(old - 60_000L, old), false)); + // no time filter -> raw + assertEquals(PostgresSmartQueryService.QueryServiceType.RAW, svc.getDelegate(Collections.emptyList(), false)); + } +} diff --git a/opennms-base-assembly/src/main/filtered/etc/vacuumd-configuration.xml b/opennms-base-assembly/src/main/filtered/etc/vacuumd-configuration.xml index 908b7bb65383..0a796c3ff2ae 100644 --- a/opennms-base-assembly/src/main/filtered/etc/vacuumd-configuration.xml +++ b/opennms-base-assembly/src/main/filtered/etc/vacuumd-configuration.xml @@ -5,11 +5,25 @@ - - DO $$ BEGIN IF EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'flow_maintain_partitions') THEN PERFORM flow_maintain_partitions(14, 3); END IF; END $$; + + DO $$ BEGIN + IF EXISTS (SELECT 1 FROM pg_proc WHERE proname = 'flow_maintain_partitions') THEN + PERFORM flow_maintain_partitions('flow', 'flow_ts', 'hour', '10 days'::interval, 48); + PERFORM flow_maintain_partitions('flow_agg', 'window_start', 'week', '6 months'::interval, 2); + END IF; + END $$; From 24868b9a5a2e41c5d2aa4b87aea6bca0d4d66571 Mon Sep 17 00:00:00 2001 From: Dino Date: Fri, 24 Jul 2026 15:11:53 -0500 Subject: [PATCH 15/15] Bump the spring bits to match develop os everything is happy --- container/features/src/main/resources/features-sentinel.xml | 2 +- container/features/src/main/resources/features.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/container/features/src/main/resources/features-sentinel.xml b/container/features/src/main/resources/features-sentinel.xml index 15e7f6fb46fb..af02970f082a 100644 --- a/container/features/src/main/resources/features-sentinel.xml +++ b/container/features/src/main/resources/features-sentinel.xml @@ -167,7 +167,7 @@ per Sentinel (aggregation.enabled=true with a distinct aggregation.writerId). Configure the flow database via the org.opennms.features.flows.persistence.postgres pid (datasource.url/username/password/databaseName). sentinel-flows - spring-jdbc + spring-jdbc opennms-core-db diff --git a/container/features/src/main/resources/features.xml b/container/features/src/main/resources/features.xml index 182b802a971c..15a359cfecbf 100644 --- a/container/features/src/main/resources/features.xml +++ b/container/features/src/main/resources/features.xml @@ -1210,7 +1210,7 @@
Optional PostgreSQL-backed FlowRepository/FlowQueryService (jsonb). Install alongside or instead of the Elasticsearch flow persistence.
opennms-flows - spring-jdbc + spring-jdbc opennms-core-db