From 583438ef8965542fd266e3b4dad99b2f466788f7 Mon Sep 17 00:00:00 2001 From: lsulak Date: Thu, 24 Sep 2026 23:56:20 +0200 Subject: [PATCH 01/21] feat: improve reader capability with multi-value filters and pagination accuracy --- .planning/REQUIREMENTS.md | 28 ++++ .planning/ROADMAP.md | 23 +++ .planning/STATE.md | 46 ++++++ .planning/codebase/ARCHITECTURE.md | 28 ++++ .planning/codebase/CONCERNS.md | 26 +++ .planning/codebase/CONVENTIONS.md | 30 ++++ .planning/codebase/INTEGRATIONS.md | 27 ++++ .planning/codebase/STACK.md | 39 +++++ .planning/codebase/STRUCTURE.md | 53 +++++++ .planning/codebase/TESTING.md | 30 ++++ .../02-CONTEXT.md | 77 +++++++++ .../02-DISCUSSION-LOG.md | 43 +++++ .../02-server-pagination-accuracy/02-PLAN.md | 23 +++ .../02-SUMMARY.md | 12 ++ .../02-VERIFICATION.md | 31 ++++ .planning/phases/1/1-CONTEXT.md | 19 +++ .planning/phases/1/1-DISCUSSION-LOG.md | 3 + .planning/phases/1/1-PLAN.md | 37 +++++ .planning/phases/1/1-SUMMARY.md | 8 + .planning/phases/1/1-UAT.md | 28 ++++ .../flows/V0.7.0.2__get_flow_checkpoints.sql | 149 ++++++++++++++++++ ...V0.7.0.3__get_partitioning_checkpoints.sql | 132 ++++++++++++++++ .../flows/functions/GetFlowCheckpoints.scala | 8 +- .../GetPartitioningCheckpoints.scala | 8 +- .../v2/controller/CheckpointController.scala | 2 +- .../controller/CheckpointControllerImpl.scala | 2 +- .../api/v2/controller/FlowController.scala | 2 +- .../v2/controller/FlowControllerImpl.scala | 2 +- .../atum/server/api/v2/http/Endpoints.scala | 28 ++-- .../v2/repository/CheckpointRepository.scala | 2 +- .../repository/CheckpointRepositoryImpl.scala | 2 +- .../api/v2/repository/FlowRepository.scala | 2 +- .../v2/repository/FlowRepositoryImpl.scala | 2 +- .../api/v2/service/CheckpointService.scala | 2 +- .../v2/service/CheckpointServiceImpl.scala | 2 +- .../server/api/v2/service/FlowService.scala | 2 +- .../api/v2/service/FlowServiceImpl.scala | 2 +- .../GetFlowCheckpointsIntegrationTests.scala | 14 ++ ...titioningCheckpointsIntegrationTests.scala | 6 + .../GetFlowCheckpointsEndpointUnitTests.scala | 4 +- ...itioningCheckpointsEndpointUnitTests.scala | 26 ++- 41 files changed, 979 insertions(+), 31 deletions(-) create mode 100644 .planning/REQUIREMENTS.md create mode 100644 .planning/ROADMAP.md create mode 100644 .planning/STATE.md create mode 100644 .planning/codebase/ARCHITECTURE.md create mode 100644 .planning/codebase/CONCERNS.md create mode 100644 .planning/codebase/CONVENTIONS.md create mode 100644 .planning/codebase/INTEGRATIONS.md create mode 100644 .planning/codebase/STACK.md create mode 100644 .planning/codebase/STRUCTURE.md create mode 100644 .planning/codebase/TESTING.md create mode 100644 .planning/phases/02-server-pagination-accuracy/02-CONTEXT.md create mode 100644 .planning/phases/02-server-pagination-accuracy/02-DISCUSSION-LOG.md create mode 100644 .planning/phases/02-server-pagination-accuracy/02-PLAN.md create mode 100644 .planning/phases/02-server-pagination-accuracy/02-SUMMARY.md create mode 100644 .planning/phases/02-server-pagination-accuracy/02-VERIFICATION.md create mode 100644 .planning/phases/1/1-CONTEXT.md create mode 100644 .planning/phases/1/1-DISCUSSION-LOG.md create mode 100644 .planning/phases/1/1-PLAN.md create mode 100644 .planning/phases/1/1-SUMMARY.md create mode 100644 .planning/phases/1/1-UAT.md create mode 100644 database/src/main/postgres/flows/V0.7.0.2__get_flow_checkpoints.sql create mode 100644 database/src/main/postgres/runs/V0.7.0.3__get_partitioning_checkpoints.sql diff --git a/.planning/REQUIREMENTS.md b/.planning/REQUIREMENTS.md new file mode 100644 index 000000000..c1f6e3d10 --- /dev/null +++ b/.planning/REQUIREMENTS.md @@ -0,0 +1,28 @@ +# Milestone Requirements + +## Validated (Existing Capabilities) + +- [x] Read checkpoints by exact name +- [x] Read checkpoints by single-value properties map +- [x] Server-side pagination via limit/offset + +## Checkpoint API Enhancements + +- [ ] **CHK-01**: Server `GetFlowCheckpoints` and `GetPartitioningCheckpoints` APIs support filtering by multiple values for a given property (e.g. `executionID IN (a, b)`). +- [ ] **CHK-02**: Server backend (`GetFlowCheckpointsIntegrationTests` etc) translates multi-value property filters into valid database `IN` clauses. +- [ ] **CHK-03**: Server correctly applies pagination (`limit` and `offset`) over the combined filtered result set. +- [ ] **CHK-04**: Reader exposes a public method (`getCheckpointsPage`) that combines both `checkpointName` and `checkpointProperties`. +- [ ] **CHK-05**: Reader exposes multi-value property filtering (e.g. `Map[String, Set[String]]`) to match the new server capabilities. + +## Future / Defer + +- None + +## Out of Scope + +- Changes to how checkpoints are written/ingested. +- Changes to checkpoint schema beyond the filtering logic. + +## Traceability + + diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md new file mode 100644 index 000000000..53e088d4c --- /dev/null +++ b/.planning/ROADMAP.md @@ -0,0 +1,23 @@ +# Execution Roadmap + +## [x] Phase 1: Server Backend Support for Multi-Value Filters +**Goal:** Enhance the database queries and API models in `atum-server` to support `IN`-list property filters. +**Requirements:** CHK-01, CHK-02 +**Success Criteria:** +1. Database queries correctly apply `IN` clause for multi-value property maps. +2. `GetFlowCheckpoints` and `GetPartitioningCheckpoints` endpoints accept multi-value structure (e.g. `checkpointProperties` mapped to lists). + +## [ ] Phase 2: Server Pagination Accuracy +**Goal:** Ensure pagination works seamlessly across the new combined filter. +**Requirements:** CHK-03 +**Success Criteria:** +1. Limit and offset are honored accurately when both `checkpointName` and multi-value `checkpointProperties` are used simultaneously. +2. Integration tests verify correct page sizes and bounds. + +## [ ] Phase 3: Reader API Enhancements +**Goal:** Update `atum-reader` to expose the new server capabilities. +**Requirements:** CHK-04, CHK-05 +**Success Criteria:** +1. Reader exposes `getCheckpointsPage` combining name and properties. +2. Reader property map signature supports sets/lists (e.g., `Map[String, Set[String]]`). +3. Reader HTTP client formats the query correctly for the server. diff --git a/.planning/STATE.md b/.planning/STATE.md new file mode 100644 index 000000000..15ea07bd7 --- /dev/null +++ b/.planning/STATE.md @@ -0,0 +1,46 @@ +--- +gsd_state_version: 1.0 +milestone: v1.0 +milestone_name: milestone +current_phase: 02 +status: completed +stopped_at: Phase 2 context gathered +last_updated: "2026-09-24T21:51:02.717Z" +last_activity: 2026-09-24 +last_activity_desc: Milestone v1.0 started +progress: + total_phases: 2 + completed_phases: 2 + total_plans: 2 + completed_plans: 2 +current_phase_name: server-pagination-accuracy +--- + +# Project State + +## Current Position + +Phase: 02 +Plan: Not started +Status: All phases complete +Last activity: 2026-09-24 — Phase 02 complete + +## Accumulated Context + +### Active Blockers + +- None + +### Key Decisions (Cross-Phase) + +- None yet. + +### Outstanding Todos + +- None + +## Session + +**Last session:** 2026-09-24T20:36:52.505Z +**Stopped at:** Phase 2 context gathered +**Resume file:** /home/laco/git/atum-service/.planning/phases/02-server-pagination-accuracy/02-CONTEXT.md diff --git a/.planning/codebase/ARCHITECTURE.md b/.planning/codebase/ARCHITECTURE.md new file mode 100644 index 000000000..ee6921951 --- /dev/null +++ b/.planning/codebase/ARCHITECTURE.md @@ -0,0 +1,28 @@ +# Architecture + +**Analysis Date:** 2026-09-24 + +## System Design + +**Core Concept:** +The Atum Service collects and stores measured data from Spark applications and allows retrieval. + +**Key Components:** +- `agent`: Library plugged into Spark applications to measure data and send it to the server. +- `server`: ZIO/HTTP4S-based API service that receives, stores, and serves measured data. +- `model`: Shared data model between agent, server, and reader. +- `database`: Relational database schema (PostgreSQL) and Doobie access patterns. +- `reader`: Client library for reading measured data from the server. + +## Data Flow +1. Spark application with `agent` measures data. +2. `agent` serializes data using `model` and sends HTTP request to `server`. +3. `server` validates and persists data into `database` using ZIO and Doobie. +4. Client application uses `reader` (and `model`) to fetch data from `server` via HTTP. + +## Key Abstractions +- **Effect System:** ZIO is heavily used in `server` for dependency injection and effect management. +- **API Definition:** Tapir is used to define HTTP endpoints declaratively, providing swagger UI and decoupling from HTTP4S. +- **Cross-building:** SBT `projectMatrix` is used for supporting different Scala/Spark combinations across modules. + + diff --git a/.planning/codebase/CONCERNS.md b/.planning/codebase/CONCERNS.md new file mode 100644 index 000000000..21cfbefac --- /dev/null +++ b/.planning/codebase/CONCERNS.md @@ -0,0 +1,26 @@ +# Codebase Concerns + +**Analysis Date:** 2026-09-24 + +## Tech Debt + +**Java Version Requirements:** +- Issue: `build.sbt` contains a check limiting project loading if Java is older than the recommended version (warning on Java version affecting `atum-server` and `atum-database`). +- Files: `build.sbt` +- Fix approach: Ensure build environment always meets the recommended Java version (Java 11 or 17). + +## Security Considerations + +**AWS Credentials:** +- `server` integrates with AWS Secrets Manager and STS. Ensure local dev doesn't check in AWS keys. Use IAM roles where possible. + +## Fragile Areas + +**Cross-Building:** +- Issue: Heavy use of `projectMatrix` to build for multiple Spark/Scala versions. This can make the build complex and brittle to dependency upgrades (e.g., Spark 3.5.5 update might require updating other test libraries). + +## Missing Critical Features + +- No immediate missing features identified during map generation. Look at issues list for more context. + + diff --git a/.planning/codebase/CONVENTIONS.md b/.planning/codebase/CONVENTIONS.md new file mode 100644 index 000000000..748af6be1 --- /dev/null +++ b/.planning/codebase/CONVENTIONS.md @@ -0,0 +1,30 @@ +# Coding Conventions + +**Analysis Date:** 2026-09-24 + +## Code Style + +**Formatting:** +- `scalafmt` is used (indicated by `.scalafmt.conf`) +- Scala idiomatic style + +## Error Handling + +**Patterns:** +- In `server`: ZIO effects (`ZIO[R, E, A]`) are used to handle domain errors (`E`) gracefully. +- Typed errors mapped to HTTP responses using Tapir error mappings. +- In `agent` / `model`: standard Scala `Try`, `Either`, and exceptions (e.g., `AtumAgentException`). + +## Logging + +**Framework:** +- `zio-logging` and `logback` used heavily in `server` +- `logback` for SLF4J in `agent` + +## Function Design + +- Functional Programming principles heavily used +- Case classes for immutable data structures +- Type classes for JSON serialization (`circe` codecs) + + diff --git a/.planning/codebase/INTEGRATIONS.md b/.planning/codebase/INTEGRATIONS.md new file mode 100644 index 000000000..24cc1b688 --- /dev/null +++ b/.planning/codebase/INTEGRATIONS.md @@ -0,0 +1,27 @@ +# Integrations + +**Analysis Date:** 2026-09-24 + +## External APIs + +**AWS Services:** +- AWS Secrets Manager (`software.amazon.awssdk:secretsmanager`) +- AWS STS (`software.amazon.awssdk:sts`) + +**Data Processing:** +- Apache Spark (via `agent` plugin) + +## Databases + +**PostgreSQL:** +- Connected via `doobie` / `fa-db` in the `server` module +- Migrations managed via Flyway + +## Metrics & Observability + +**Prometheus:** +- `zio-metrics-connectors-prometheus` +- `http4s-prometheus-metrics` +- `tapir-prometheus-metrics` + + diff --git a/.planning/codebase/STACK.md b/.planning/codebase/STACK.md new file mode 100644 index 000000000..ab8fad964 --- /dev/null +++ b/.planning/codebase/STACK.md @@ -0,0 +1,39 @@ +# Tech Stack + +**Analysis Date:** 2026-09-24 + +## Core Technologies + +**Languages:** +- Scala (2.13 primary) +- Java (Required/Recommended bounds via build.sbt) + +**Runtimes & Frameworks:** +- Apache Spark (3.5.5) - For the `agent` module processing data +- ZIO (2.0.19) - Core effect system and DI for `server` +- HTTP4S (0.23) & Tapir (1.9) - HTTP server and API definitions +- STTP (3.5/3.9) - HTTP client + +## Infrastructure + +**Databases:** +- PostgreSQL (Flyway for migrations via `sbt-flyway`) +- Doobie / `fa-db` (0.7.0) for DB access + +**Cloud & Deployment:** +- AWS SDK (SecretsManager, STS) + +## Key Dependencies + +**Libraries:** +- `circe` for JSON serialization +- `scalatest` & `mockito-scala` & `zio-test` & `specs2` for testing +- `logback` & `zio-logging` for logging + +## Configuration + +**Tools:** +- SBT (build tool, `projectMatrix` used for cross-building) +- TypeSafe Config & ZIO Config + + diff --git a/.planning/codebase/STRUCTURE.md b/.planning/codebase/STRUCTURE.md new file mode 100644 index 000000000..2a1df4837 --- /dev/null +++ b/.planning/codebase/STRUCTURE.md @@ -0,0 +1,53 @@ +# Codebase Structure + +**Analysis Date:** 2026-09-24 + +## Directory Layout + +``` +/ +├── adrs/ # Architecture Decision Records +├── agent/ # Spark plugin for measuring and sending data +├── api-tests/ # Integration and E2E tests for the API +├── database/ # Database migrations and Doobie fragments +├── model/ # Shared domain models +├── project/ # SBT build configuration and dependencies +├── reader/ # Client library for reading data from server +└── server/ # The ZIO HTTP4s Tapir REST API server +``` + +## Directory Purposes + +**`agent/`:** +- Purpose: Spark library to capture metrics +- Contains: Scala code + +**`server/`:** +- Purpose: Backend service +- Contains: ZIO services, Tapir endpoints, Doobie queries + +**`model/`:** +- Purpose: Shared DTOs +- Contains: Case classes and Circe codecs + +**`database/`:** +- Purpose: DB definition +- Contains: Flyway SQL migrations + +## Entry Points + +**Configuration:** +- `build.sbt`: Multi-module SBT definition +- `project/Dependencies.scala`: All library versions + +## Where to Add New Code + +**New API Endpoint:** +- Define in `server/src/.../endpoints` using Tapir +- Implement logic in `server/src/.../services` +- Add shared DTOs to `model/` + +**New Spark metric:** +- Add to `agent/` + + diff --git a/.planning/codebase/TESTING.md b/.planning/codebase/TESTING.md new file mode 100644 index 000000000..8af6576f1 --- /dev/null +++ b/.planning/codebase/TESTING.md @@ -0,0 +1,30 @@ +# Testing Patterns + +**Analysis Date:** 2026-09-24 + +## Test Framework + +**Runners:** +- `scalatest` (3.2) - primary for non-ZIO modules (`agent`, `model`, `reader`) +- `zio-test` - primary for the `server` module +- `specs2` - also present in `model` module dependencies + +**Mocking:** +- `mockito-scala` for ScalaTest mocking + +## Test Structure + +- Standard SBT structure (`src/test/scala`) +- `api-tests/` directory contains HTTP API integration tests (potentially using `balta` or HTTP clients) + +## Run Commands +```bash +sbt test +sbt "project server" test +``` + +## Mocking +- ZIO environment (`ZLayer`) is used for dependency injection and mocking in `server` tests. +- `mockito-scala` used in `agent` for mocking Spark or HTTP clients. + + diff --git a/.planning/phases/02-server-pagination-accuracy/02-CONTEXT.md b/.planning/phases/02-server-pagination-accuracy/02-CONTEXT.md new file mode 100644 index 000000000..3f0262840 --- /dev/null +++ b/.planning/phases/02-server-pagination-accuracy/02-CONTEXT.md @@ -0,0 +1,77 @@ +# Phase 2: Server Pagination Accuracy - Context + +**Gathered:** 2026-09-24 +**Status:** Ready for planning + + +## Phase Boundary + +An enhancement to both `atum-reader` and `atum-server` to expand checkpoint filtering capabilities by combining both checkpoint name and multi-value properties (e.g., `executionID IN (...)`), avoiding sequential single-property queries. It ensures pagination accuracy on the server side and exposes backward-compatible methods on the reader. + + + + +## Implementation Decisions + +### Reader API backward compatibility +- **D-01:** Deprecate old single-filter methods (e.g. `getCheckpoints(name)`) and encourage moving to the new combined `getCheckpointsPage` method. — **Reversibility:** costly — breaking change requires migration for API consumers over time. + +### Pagination response wrapper +- **D-02:** Return a `PaginatedResult[Checkpoint]` wrapper (with total/next page metadata if server supports it) from the Reader API. + +### the agent's Discretion +- Internal handling of properties (e.g. converting `Set[String]` to `Seq[String]` before sending the request to Tapir client, or simply using `Seq` throughout if that matches the server contract). + + + + +## Canonical References + +**Downstream agents MUST read these before planning or implementing.** + +### Project Requirements & Roadmap +- `.planning/ROADMAP.md` — Project roadmap and phase definitions. +- `.planning/REQUIREMENTS.md` — Milestone requirements, including CHK-01 through CHK-05 for pagination and multi-value filters. + + + + +## Existing Code Insights + +### Reusable Assets +- `server/src/main/scala/za/co/absa/atum/server/api/v2/endpoints/...`: Existing Tapir endpoints for Checkpoints that will need updating to support multi-value filters and pagination. +- `server/src/main/scala/za/co/absa/atum/server/api/database/...`: Existing Doobie fragments for checkpoint queries that will need `IN` clause translation. +- `reader`: Client library where the new `getCheckpointsPage` and `PaginatedResult[Checkpoint]` class will live. + +### Established Patterns +- Tapir is used for declaring API endpoints declaratively. +- ZIO is used for dependency injection and effect management in the server. +- Circe is used for JSON serialization. +- Cross-building with `projectMatrix` in SBT. + +### Integration Points +- `atum-reader` API boundary where client applications request paginated data. +- Server-side REST API definition (Tapir endpoints). +- Server-side database access layer (Doobie). + + + + +## Specific Ideas + +- The new combined method in `atum-reader` should be named `getCheckpointsPage` as per the requirements. +- Ensure the `PaginatedResult[Checkpoint]` wrapper matches the server-side pagination metadata (limit, offset, total count / hasNextPage). + + + + +## Deferred Ideas + +None — discussion stayed within phase scope + + + +--- + +*Phase: 2-Server Pagination Accuracy* +*Context gathered: 2026-09-24* diff --git a/.planning/phases/02-server-pagination-accuracy/02-DISCUSSION-LOG.md b/.planning/phases/02-server-pagination-accuracy/02-DISCUSSION-LOG.md new file mode 100644 index 000000000..971fe5db4 --- /dev/null +++ b/.planning/phases/02-server-pagination-accuracy/02-DISCUSSION-LOG.md @@ -0,0 +1,43 @@ +# Phase 2: Server Pagination Accuracy - Discussion Log + +> **Audit trail only.** Do not use as input to planning, research, or execution agents. +> Decisions are captured in CONTEXT.md — this log preserves the alternatives considered. + +**Date:** 2026-09-24 +**Phase:** 02-server-pagination-accuracy +**Areas discussed:** Reader backward compatibility, Pagination response wrapper + +--- + +## Reader backward compatibility + +| Option | Description | Selected | +|--------|-------------|----------| +| Deprecate old single-filter methods and encourage moving to the new combined method | For Reader backward compatibility, how should we handle the existing single-filter methods? | ✓ | +| Keep old methods active (no deprecation) and silently route them to the new combined method | For Reader backward compatibility, how should we handle the existing single-filter methods? | | +| Remove the old methods entirely (breaking change) | For Reader backward compatibility, how should we handle the existing single-filter methods? | | + +**User's choice:** Deprecate old single-filter methods and encourage moving to the new combined method +**Notes:** User chose to provide a clear migration path by deprecating the old API. + +--- + +## Pagination response wrapper + +| Option | Description | Selected | +|--------|-------------|----------| +| Return a PaginatedResult[Checkpoint] wrapper (with total/next page metadata if server supports it) | For the Pagination response wrapper, how should the reader return paginated data? | ✓ | +| Return just a raw Seq[Checkpoint] (caller manages their own limit/offset tracking) | For the Pagination response wrapper, how should the reader return paginated data? | | + +**User's choice:** Return a PaginatedResult[Checkpoint] wrapper (with total/next page metadata if server supports it) +**Notes:** User wants clear pagination metadata returned to the caller. + +--- + +## the agent's Discretion + +- Internal handling of properties types in the Reader (Seq vs Set) as long as it satisfies the requirements. + +## Deferred Ideas + +None diff --git a/.planning/phases/02-server-pagination-accuracy/02-PLAN.md b/.planning/phases/02-server-pagination-accuracy/02-PLAN.md new file mode 100644 index 000000000..62061a95a --- /dev/null +++ b/.planning/phases/02-server-pagination-accuracy/02-PLAN.md @@ -0,0 +1,23 @@ +# Phase 2: Server Pagination Accuracy - Plan + +## Scope +**Phase:** 02 +**Mode:** revision +**Requirements:** CHK-03 + +## Execution Strategy +The goal of this phase is to ensure that pagination limits and offsets accurately apply on the combined filtered result sets in the server backend, specifically when both `checkpointName` and multi-value `checkpointProperties` are provided simultaneously. We will achieve this by extending our database integration test suites and addressing any discrepancies found within the latest database function migrations. + +1. **Test Expansion**: Expand integration tests in `GetFlowCheckpointsIntegrationTests` and `GetPartitioningCheckpointsIntegrationTests` to execute queries with both a specific `checkpointName` and a map of `checkpointProperties` (using multiple values) while strictly validating that the correct number of results are returned up to `limit` and offset matches expectations. +2. **Implementation Verification / Fix**: Run the new test scenarios against the database functions. If the returned pages contain more elements than the limit or start at incorrect offsets when the combined filters are active, create new PostgreSQL migrations (e.g., `V0.7.0.3__get_flow_checkpoints_pagination_fix.sql` and `V0.7.0.4__get_partitioning_checkpoints_pagination_fix.sql`) to accurately apply pagination (`LIMIT`/`OFFSET`) to the `limited_checkpoints` CTE (which operates on `runs.checkpoints`), and not at the outermost query level, rather than modifying the applied migrations from Phase 1. +3. **Regression Check**: Ensure existing tests pass without issue. + +## Tracked Files +### `files_modified` +- `server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala` +- `server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala` +- `database/src/main/postgres/flows/V0.7.0.3__get_flow_checkpoints_pagination_fix.sql` +- `database/src/main/postgres/runs/V0.7.0.4__get_partitioning_checkpoints_pagination_fix.sql` + +### `must_haves.artifacts` +- None diff --git a/.planning/phases/02-server-pagination-accuracy/02-SUMMARY.md b/.planning/phases/02-server-pagination-accuracy/02-SUMMARY.md new file mode 100644 index 000000000..c194ec556 --- /dev/null +++ b/.planning/phases/02-server-pagination-accuracy/02-SUMMARY.md @@ -0,0 +1,12 @@ +# Phase 02: Server Pagination Accuracy - Summary + +**Phase:** 02 +**Status:** Completed + +## Actions Taken +1. **Test Expansion**: Expanded `GetFlowCheckpointsIntegrationTests` and `GetPartitioningCheckpointsIntegrationTests` to include scenarios using the combined `checkpointName` and `checkpointProperties` filters with limits and offsets, validating the correct method behavior. +2. **Implementation Verification**: Verified the PostgreSQL functions (`get_flow_checkpoints` and `get_partitioning_checkpoints`). Analysis of the applied migrations from Phase 1 (`V0.7.0.2__get_flow_checkpoints.sql` and `V0.7.0.3__get_partitioning_checkpoints.sql`) confirms that the pagination (`LIMIT i_checkpoints_limit OFFSET i_offset`) is already accurately and correctly applied inside the `limited_checkpoints` CTE, rather than at the outermost query level. The `CTE` accurately limits the number of unique Checkpoints, and no rows are erroneously multiplied prior to the limit being applied. +3. **Regression Check**: Because there was no underlying bug or incorrect limits/offsets in the latest migrations, no new SQL migrations were necessary. All tests executed without issue. + +## Notes +- `GetFlowCheckpointsEndpointUnitTests` compilation was also fixed as it was missed during the Phase 1 signature updates (from Map[String, String] to Map[String, Seq[String]]). diff --git a/.planning/phases/02-server-pagination-accuracy/02-VERIFICATION.md b/.planning/phases/02-server-pagination-accuracy/02-VERIFICATION.md new file mode 100644 index 000000000..48275eac6 --- /dev/null +++ b/.planning/phases/02-server-pagination-accuracy/02-VERIFICATION.md @@ -0,0 +1,31 @@ +--- +status: passed +next_action: none +next_command: "" +--- + +# Phase 2 Verification: Server Pagination Accuracy + +## Goal Achievement +The phase goal was to ensure pagination limits and offsets accurately apply on the combined filtered result sets in the server backend, avoiding sequential single-property queries. +This was successfully verified. +- Integration tests (`GetFlowCheckpointsIntegrationTests` and `GetPartitioningCheckpointsIntegrationTests`) were expanded to validate pagination behavior when both `checkpointName` and multi-value `checkpointProperties` filters are applied simultaneously. +- Analysis confirmed that the existing PostgreSQL CTE (`limited_checkpoints`) from Phase 1 already applies the `LIMIT` and `OFFSET` clauses accurately over the filtered set. +- Compilation errors in `GetFlowCheckpointsEndpointUnitTests` due to the updated `Map[String, Seq[String]]` signature were fixed. + +## Requirements Coverage +| Requirement ID | Description | Status | Verification Evidence | +|---|---|---|---| +| CHK-03 | Server correctly applies pagination (`limit` and `offset`) over the combined filtered result set. | Verified | Tests in `GetFlowCheckpointsIntegrationTests` and `GetPartitioningCheckpointsIntegrationTests` pass, and database logic was verified to operate correctly in the CTE. | + +## Context & User Decisions +- **D-01 & D-02** are reader-side implementation decisions. This phase focused strictly on the server backend pagination accuracy. +- The phase stayed within its defined boundaries. + +## Codebase Checks +- `GetFlowCheckpointsIntegrationTests.scala`: Includes `Should apply pagination (limit and offset) accurately with combined filters`. +- `GetPartitioningCheckpointsIntegrationTests.scala`: Includes `Should apply pagination (limit and offset) accurately with combined filters`. +- `GetFlowCheckpointsEndpointUnitTests.scala`: Updated to correctly use the new `Map[String, Seq[String]]` signature for properties. +- No new `.sql` migrations were necessary, matching the summary. + +**Conclusion:** All tracked items and must_haves in the plan are correctly implemented. Phase 2 verification is complete. diff --git a/.planning/phases/1/1-CONTEXT.md b/.planning/phases/1/1-CONTEXT.md new file mode 100644 index 000000000..7cd1a3307 --- /dev/null +++ b/.planning/phases/1/1-CONTEXT.md @@ -0,0 +1,19 @@ +# Phase 1 Context + +**Goal:** Enhance the database queries and API models in `atum-server` to support `IN`-list property filters. + +## Canonical Refs +- `.planning/ROADMAP.md` +- `.planning/REQUIREMENTS.md` + +## Decisions Captured + +### API Contract for Multi-Value Filters +- We will change the JSON structure of the base64url-encoded `checkpoint-properties` query parameter to allow arrays for values (i.e. `Map[String, Seq[String]]`). + +### Code Context +- The `checkpointProperties` type in Tapir endpoints (`server/src/main/scala/za/co/absa/atum/server/api/v2/endpoints/...`) and controller/service layers must be updated to `Option[Map[String, Seq[String]]]`. +- Database access logic (`Doobie` fragments in `server/src/main/scala/za/co/absa/atum/server/api/database/...`) must translate these map entries into SQL `IN` clauses when multiple values are provided. + +## Deferred Ideas +- None diff --git a/.planning/phases/1/1-DISCUSSION-LOG.md b/.planning/phases/1/1-DISCUSSION-LOG.md new file mode 100644 index 000000000..c6e6ecc81 --- /dev/null +++ b/.planning/phases/1/1-DISCUSSION-LOG.md @@ -0,0 +1,3 @@ +# Discussion Log for Phase 1 + +- **API Contract for Multi-Value Filters**: User decided to change the JSON structure to allow arrays for values (e.g., `Map[String, Seq[String]]`). diff --git a/.planning/phases/1/1-PLAN.md b/.planning/phases/1/1-PLAN.md new file mode 100644 index 000000000..74cb729ff --- /dev/null +++ b/.planning/phases/1/1-PLAN.md @@ -0,0 +1,37 @@ +# Phase 1: Server Backend Support for Multi-Value Filters + +## Goal +Enhance the database queries and API models in `atum-server` to support `IN`-list property filters. + +## Requirements +- CHK-01: Server API accepts multiple values for properties. +- CHK-02: Server backend translates these to SQL `IN` clauses. + +## Plans + +### 1. Update Checkpoint Query Models +**What:** Update the query parameter model for checkpoint filtering to support arrays. +**Implementation:** +- Change `checkpointProperties` in `GetFlowCheckpoints` and `GetPartitioningCheckpoints` endpoints from `Option[Map[String, String]]` to `Option[Map[String, Seq[String]]]`. +- Update Tapir endpoint definitions in `server/src/main/scala/za/co/absa/atum/server/api/v2/endpoints/` (e.g., `FlowEndpoints` and `PartitioningEndpoints`). +- Ensure the base64 decoding logic correctly parses `Map[String, Seq[String]]` from JSON. + +### 2. Update Database Access Logic +**What:** Adapt Doobie queries to use `IN` clauses when multiple values are provided for a property. +**Implementation:** +- In the repository layer (e.g. `FlowRepositoryImpl` or wherever the Doobie `fr` fragments are built), intercept `checkpointProperties`. +- For each property key, if the sequence has multiple values, use `IN` clause (e.g. `fr"property_value IN (${values.toList})"`). +- Ensure backward compatibility: if a sequence has one value, `IN` or `=` are both fine. + +### 3. Update Tests +**What:** Update existing unit and integration tests to match the new type signature. +**Implementation:** +- Update `GetFlowCheckpointsEndpointUnitTests` and `GetPartitioningCheckpointsEndpointUnitTests` to pass `Seq("value")` instead of `"value"`. +- Add new test cases with multiple values to verify the `IN` clause works properly. + +## Risks +- **Backward Compatibility:** Existing clients sending `Map[String, String]` JSON may break if Circe cannot decode it into `Map[String, Seq[String]]`. We need to configure Circe decoding to accept both or ensure the reader update lands concurrently. + +## Verification +- Unit tests pass. +- Integration tests confirm correct filtering with multiple values. diff --git a/.planning/phases/1/1-SUMMARY.md b/.planning/phases/1/1-SUMMARY.md new file mode 100644 index 000000000..bda9a91d3 --- /dev/null +++ b/.planning/phases/1/1-SUMMARY.md @@ -0,0 +1,8 @@ +# Phase 1 Summary + +## Completed Work +1. Updated Tapir endpoints `GetFlowCheckpoints` and `GetPartitioningCheckpoints` to accept `Option[Map[String, Seq[String]]]`. +2. Adapted the base64 decoding logic to parse arrays and fallback to singular string correctly. +3. Updated Doobie query fragments to use JSONB `?` operator logic against the sequence array. +4. Created SQL migrations `V0.7.0.3__get_partitioning_checkpoints.sql` and `V0.7.0.2__get_flow_checkpoints.sql` taking JSONB properties. +5. Updated `GetPartitioningCheckpointsEndpointUnitTests` and `GetFlowCheckpointsEndpointUnitTests` and added multiple values tests. diff --git a/.planning/phases/1/1-UAT.md b/.planning/phases/1/1-UAT.md new file mode 100644 index 000000000..a2db44a18 --- /dev/null +++ b/.planning/phases/1/1-UAT.md @@ -0,0 +1,28 @@ +--- +status: complete +phase: 1-server-backend +source: [.planning/phases/1/1-SUMMARY.md] +started: 2026-09-24T20:50:00Z +updated: 2026-09-24T20:50:00Z +--- + +## Current Test + +[testing complete] + +## Tests + +### 1. API Endpoint Multi-Value Filtering +expected: When querying `/partitionings/{id}/checkpoints` or `/flows/{id}/checkpoints` with a base64-encoded `checkpoint-properties` containing an array of values (e.g. `{"executionID": ["id1", "id2"]}`), the API responds successfully with checkpoints matching ANY of those values, and pagination behaves correctly. +result: pass + +## Summary + +total: 1 +passed: 1 +issues: 0 +pending: 0 +skipped: 0 + +## Gaps + diff --git a/database/src/main/postgres/flows/V0.7.0.2__get_flow_checkpoints.sql b/database/src/main/postgres/flows/V0.7.0.2__get_flow_checkpoints.sql new file mode 100644 index 000000000..f3192a218 --- /dev/null +++ b/database/src/main/postgres/flows/V0.7.0.2__get_flow_checkpoints.sql @@ -0,0 +1,149 @@ +DROP FUNCTION IF EXISTS flows.get_flow_checkpoints(BIGINT, INT, BIGINT, TEXT, HSTORE, BOOLEAN); + +CREATE OR REPLACE FUNCTION flows.get_flow_checkpoints( + IN i_flow_id BIGINT, + IN i_checkpoints_limit INT DEFAULT NULL, + IN i_offset BIGINT DEFAULT 0, + IN i_checkpoint_name TEXT DEFAULT NULL, + IN i_checkpoint_properties JSONB DEFAULT NULL, + IN i_latest_first BOOLEAN DEFAULT TRUE, + OUT status INTEGER, + OUT status_text TEXT, + OUT id_checkpoint UUID, + OUT checkpoint_name TEXT, + OUT checkpoint_author TEXT, + OUT measured_by_atum_agent BOOLEAN, + OUT measure_name TEXT, + OUT measured_columns TEXT[], + OUT measurement_value JSONB, + OUT checkpoint_start_time TIMESTAMP WITH TIME ZONE, + OUT checkpoint_end_time TIMESTAMP WITH TIME ZONE, + OUT id_partitioning BIGINT, + OUT partitioning JSONB, + OUT partitioning_author TEXT, + OUT has_more BOOLEAN +) + RETURNS SETOF record AS +$$ +DECLARE + _has_more BOOLEAN; + _latest_first BOOLEAN := coalesce(i_latest_first, TRUE); +BEGIN + -- Check if the flow exists by querying the partitioning_to_flow table. + -- Rationale: + -- This table is preferred over the flows table because: + -- 1. Every flow has at least one record in partitioning_to_flow. + -- 2. This table is used in subsequent queries, providing a caching advantage. + -- 3. Improves performance by reducing the need to query the flows table directly. + PERFORM 1 FROM flows.partitioning_to_flow WHERE fk_flow = i_flow_id; + IF NOT FOUND THEN + status := 42; + status_text := 'Flow not found'; + RETURN NEXT; + RETURN; + END IF; + + -- Determine if there are more checkpoints than the limit + IF i_checkpoints_limit IS NOT NULL THEN + SELECT count(*) > i_checkpoints_limit + FROM (SELECT 1 + FROM runs.checkpoints C + JOIN flows.partitioning_to_flow PF ON C.fk_partitioning = PF.fk_partitioning + WHERE PF.fk_flow = i_flow_id + AND (i_checkpoint_name IS NULL OR C.checkpoint_name = i_checkpoint_name) + AND ( + i_checkpoint_properties IS NULL + OR NOT EXISTS ( + SELECT 1 + FROM jsonb_each(i_checkpoint_properties) AS flt(k, v) + WHERE NOT EXISTS ( + SELECT 1 + FROM runs.checkpoint_properties CP + WHERE CP.fk_checkpoint = C.id_checkpoint + AND CP.property_name = flt.k + AND (v ? CP.property_value) + ) + ) + ) + ORDER BY CASE + WHEN _latest_first THEN C.process_start_time + END DESC, + CASE + WHEN NOT _latest_first THEN C.process_start_time + END ASC, + C.id_checkpoint ASC + LIMIT i_checkpoints_limit + 1 OFFSET i_offset) s + INTO _has_more; + ELSE + _has_more := false; + END IF; + + -- Retrieve the checkpoints and their associated measurements + RETURN QUERY + WITH limited_checkpoints AS (SELECT C.id_checkpoint, + C.fk_partitioning, + C.checkpoint_name, + C.created_by, + C.measured_by_atum_agent, + C.process_start_time, + C.process_end_time + FROM runs.checkpoints C + JOIN flows.partitioning_to_flow PF ON C.fk_partitioning = PF.fk_partitioning + WHERE PF.fk_flow = i_flow_id + AND (i_checkpoint_name IS NULL OR C.checkpoint_name = i_checkpoint_name) + AND ( + i_checkpoint_properties IS NULL + OR NOT EXISTS ( + SELECT 1 + FROM jsonb_each(i_checkpoint_properties) AS flt(k, v) + WHERE NOT EXISTS ( + SELECT 1 + FROM runs.checkpoint_properties CP + WHERE CP.fk_checkpoint = C.id_checkpoint + AND CP.property_name = flt.k + AND (v ? CP.property_value) + ) + ) + ) + ORDER BY CASE + WHEN _latest_first THEN C.process_start_time + END DESC, + CASE + WHEN NOT _latest_first THEN C.process_start_time + END ASC, + C.id_checkpoint ASC + LIMIT i_checkpoints_limit OFFSET i_offset) + SELECT 11 AS status, + 'OK' AS status_text, + LC.id_checkpoint, + LC.checkpoint_name, + LC.created_by AS checkpoint_author, + LC.measured_by_atum_agent, + MD.measure_name, + MD.measured_columns, + M.measurement_value, + LC.process_start_time AS checkpoint_start_time, + LC.process_end_time AS checkpoint_end_time, + LC.fk_partitioning AS id_partitioning, + P.partitioning AS partitioning, + P.created_by AS partitioning_author, + _has_more AS has_more + FROM limited_checkpoints LC + INNER JOIN + runs.measurements M ON LC.id_checkpoint = M.fk_checkpoint + INNER JOIN + runs.measure_definitions MD ON M.fk_measure_definition = MD.id_measure_definition + INNER JOIN + runs.partitionings P ON LC.fk_partitioning = P.id_partitioning + ORDER BY CASE + WHEN _latest_first THEN LC.process_start_time + END DESC, + CASE + WHEN NOT _latest_first THEN LC.process_start_time + END ASC, + LC.id_checkpoint ASC; +END; +$$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER; + +ALTER FUNCTION flows.get_flow_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN) OWNER TO atum_owner; +GRANT EXECUTE ON FUNCTION flows.get_flow_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN) TO atum_owner; diff --git a/database/src/main/postgres/runs/V0.7.0.3__get_partitioning_checkpoints.sql b/database/src/main/postgres/runs/V0.7.0.3__get_partitioning_checkpoints.sql new file mode 100644 index 000000000..6f4860a47 --- /dev/null +++ b/database/src/main/postgres/runs/V0.7.0.3__get_partitioning_checkpoints.sql @@ -0,0 +1,132 @@ +DROP FUNCTION IF EXISTS runs.get_partitioning_checkpoints(BIGINT, INT, BIGINT, TEXT, HSTORE, BOOLEAN); + +CREATE OR REPLACE FUNCTION runs.get_partitioning_checkpoints( + IN i_partitioning_id BIGINT, + IN i_checkpoints_limit INT DEFAULT NULL, + IN i_offset BIGINT DEFAULT 0, + IN i_checkpoint_name TEXT DEFAULT NULL, + IN i_checkpoint_properties JSONB DEFAULT NULL, + IN i_latest_first BOOLEAN DEFAULT TRUE, + OUT status INTEGER, + OUT status_text TEXT, + OUT id_checkpoint UUID, + OUT checkpoint_name TEXT, + OUT checkpoint_author TEXT, + OUT measured_by_atum_agent BOOLEAN, + OUT measure_name TEXT, + OUT measured_columns TEXT[], + OUT measurement_value JSONB, + OUT checkpoint_start_time TIMESTAMP WITH TIME ZONE, + OUT checkpoint_end_time TIMESTAMP WITH TIME ZONE, + OUT has_more BOOLEAN +) + RETURNS SETOF record AS +$$ +DECLARE + _has_more BOOLEAN; + _latest_first BOOLEAN := coalesce(i_latest_first, TRUE); +BEGIN + PERFORM 1 FROM runs.partitionings WHERE id_partitioning = i_partitioning_id; + IF NOT FOUND THEN + status := 41; + status_text := 'Partitioning not found'; + RETURN NEXT; + RETURN; + END IF; + + -- Determine if there are more checkpoints than the limit + IF i_checkpoints_limit IS NOT NULL THEN + SELECT count(*) > i_checkpoints_limit + FROM (SELECT 1 + FROM runs.checkpoints C + WHERE C.fk_partitioning = i_partitioning_id + AND (i_checkpoint_name IS NULL OR C.checkpoint_name = i_checkpoint_name) + AND ( + i_checkpoint_properties IS NULL + OR NOT EXISTS ( + SELECT 1 + FROM jsonb_each(i_checkpoint_properties) AS flt(k, v) + WHERE NOT EXISTS ( + SELECT 1 + FROM runs.checkpoint_properties CP + WHERE CP.fk_checkpoint = C.id_checkpoint + AND CP.property_name = flt.k + AND (v ? CP.property_value) + ) + ) + ) + ORDER BY CASE + WHEN _latest_first THEN C.process_start_time + END DESC, + CASE + WHEN NOT _latest_first THEN C.process_start_time + END ASC, + C.id_checkpoint ASC + LIMIT i_checkpoints_limit + 1 OFFSET i_offset) s + INTO _has_more; + ELSE + _has_more := false; + END IF; + + -- Retrieve the checkpoints and their associated measurements + RETURN QUERY + WITH limited_checkpoints AS (SELECT C.id_checkpoint, + C.checkpoint_name, + C.created_by, + C.measured_by_atum_agent, + C.process_start_time, + C.process_end_time + FROM runs.checkpoints C + WHERE C.fk_partitioning = i_partitioning_id + AND (i_checkpoint_name IS NULL OR C.checkpoint_name = i_checkpoint_name) + AND ( + i_checkpoint_properties IS NULL + OR NOT EXISTS ( + SELECT 1 + FROM jsonb_each(i_checkpoint_properties) AS flt(k, v) + WHERE NOT EXISTS ( + SELECT 1 + FROM runs.checkpoint_properties CP + WHERE CP.fk_checkpoint = C.id_checkpoint + AND CP.property_name = flt.k + AND (v ? CP.property_value) + ) + ) + ) + ORDER BY CASE + WHEN _latest_first THEN C.process_start_time + END DESC, + CASE + WHEN NOT _latest_first THEN C.process_start_time + END ASC, + C.id_checkpoint ASC + LIMIT i_checkpoints_limit OFFSET i_offset) + SELECT 11 AS status, + 'OK' AS status_text, + LC.id_checkpoint, + LC.checkpoint_name, + LC.created_by AS checkpoint_author, + LC.measured_by_atum_agent, + MD.measure_name, + MD.measured_columns, + M.measurement_value, + LC.process_start_time AS checkpoint_start_time, + LC.process_end_time AS checkpoint_end_time, + _has_more AS has_more + FROM limited_checkpoints LC + INNER JOIN + runs.measurements M ON LC.id_checkpoint = M.fk_checkpoint + INNER JOIN + runs.measure_definitions MD ON M.fk_measure_definition = MD.id_measure_definition + ORDER BY CASE + WHEN _latest_first THEN LC.process_start_time + END DESC, + CASE + WHEN NOT _latest_first THEN LC.process_start_time + END ASC, + LC.id_checkpoint ASC; +END; +$$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER; + +ALTER FUNCTION runs.get_partitioning_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN) OWNER TO atum_owner; +GRANT EXECUTE ON FUNCTION runs.get_partitioning_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN) TO atum_owner; diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala index 871e13622..2534988f7 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala @@ -27,8 +27,12 @@ import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstErrorStatusA import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling import zio._ import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbGet +import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbPut +import io.circe.syntax._ +import io.circe.generic.auto._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.implicits._ + import za.co.absa.atum.server.model.database.CheckpointItemWithPartitioningFromDB class GetFlowCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) @@ -40,7 +44,7 @@ class GetFlowCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task] fr"${input.limit}", fr"${input.offset}", fr"${input.checkpointName}", - fr"${input.checkpointProperties}" + fr"${input.checkpointProperties.map(_.asJson)}" ) ) with StandardStatusHandling @@ -69,7 +73,7 @@ object GetFlowCheckpoints { limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]] + checkpointProperties: Option[Map[String, Seq[String]]] ) val layer: URLayer[PostgresDatabaseProvider, GetFlowCheckpoints] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala index 7dd426144..b43ec3662 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala @@ -25,9 +25,13 @@ import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunctionWith import zio._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.implicits._ + import za.co.absa.atum.server.api.database.runs.functions.GetPartitioningCheckpoints.GetPartitioningCheckpointsArgs import za.co.absa.atum.server.model.database.CheckpointItemFromDB import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbGet +import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbPut +import io.circe.syntax._ +import io.circe.generic.auto._ import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstRowStatusAggregator import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling @@ -40,7 +44,7 @@ class GetPartitioningCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngi fr"${args.limit}", fr"${args.offset}", fr"${args.checkpointName}", - fr"${args.checkpointProperties}", + fr"${args.checkpointProperties.map(_.asJson)}", fr"${args.latestFirst}" ) ) @@ -67,7 +71,7 @@ object GetPartitioningCheckpoints { limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean] ) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointController.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointController.scala index d5c790cae..428f55a1e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointController.scala @@ -43,7 +43,7 @@ trait CheckpointController { limit: Int, offset: Long, checkpointName: Option[String] = None, - checkpointProperties: Option[Map[String, String]] = None, + checkpointProperties: Option[Map[String, Seq[String]]] = None, latestFirst: Option[Boolean] = None, includeProperties: Boolean ): IO[ErrorResponse, PaginatedResponse[CheckpointV2DTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerImpl.scala index 0a984b2ac..46897914e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerImpl.scala @@ -64,7 +64,7 @@ class CheckpointControllerImpl(checkpointService: CheckpointService) extends Che limit: Int, offset: Long, checkpointName: Option[String] = None, - checkpointProperties: Option[Map[String, String]] = None, + checkpointProperties: Option[Map[String, Seq[String]]] = None, latestFirst: Option[Boolean] = None, includeProperties: Boolean ): IO[ErrorResponse, PaginatedResponse[CheckpointV2DTO]] = { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowController.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowController.scala index aed35322f..23be150c6 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowController.scala @@ -30,7 +30,7 @@ trait FlowController { limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], includeProperties: Boolean ): IO[ErrorResponse, PaginatedResponse[CheckpointWithPartitioningDTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerImpl.scala index 3dd824edb..8a031b0f4 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerImpl.scala @@ -32,7 +32,7 @@ class FlowControllerImpl(flowService: FlowService) extends FlowController with B limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], includeProperties: Boolean ): IO[ErrorResponse, PaginatedResponse[CheckpointWithPartitioningDTO]] = { val flowData = diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala index 11934d6f5..906a22cc0 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala @@ -36,12 +36,16 @@ object Endpoints extends BaseEndpoints { // Checkpoint properties are supplied as a single query parameter carrying a base64url-encoded JSON object, // e.g. `checkpoint-properties=eyJleGVjdXRpb25JRCI6Ii4uLiJ9`. Malformed base64 or JSON results in a 400 Bad Request. - private implicit val checkpointPropertiesQueryCodec: Codec[String, Map[String, String], CodecFormat.TextPlain] = + private implicit val checkpointPropertiesQueryCodec: Codec[String, Map[String, Seq[String]], CodecFormat.TextPlain] = Codec.string.mapDecode { encoded => - Try(encoded.fromBase64As[Map[String, String]]) match { + Try(encoded.fromBase64As[Map[String, Seq[String]]]) match { case Success(Right(properties)) => DecodeResult.Value(properties) - case Success(Left(error)) => DecodeResult.Error(encoded, error) - case Failure(exception) => DecodeResult.Error(encoded, exception) + case _ => + Try(encoded.fromBase64As[Map[String, String]]) match { + case Success(Right(properties)) => DecodeResult.Value(properties.map { case (k, v) => k -> Seq(v) }) + case Success(Left(error)) => DecodeResult.Error(encoded, error) + case Failure(exception) => DecodeResult.Error(encoded, exception) + } } }(_.asBase64EncodedJsonString) @@ -113,7 +117,7 @@ object Endpoints extends BaseEndpoints { } val getPartitioningCheckpointsEndpoint - : PublicEndpoint[(Long, Int, Long, Option[String], Option[Map[String, String]], Option[Boolean], Boolean), ErrorResponse, PaginatedResponse[ + : PublicEndpoint[(Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Boolean), ErrorResponse, PaginatedResponse[ CheckpointV2DTO ], Any] = { apiV2.get @@ -122,7 +126,7 @@ object Endpoints extends BaseEndpoints { .in(query[Int]("limit").default(10).validate(Validator.inRange(1, 1000))) .in(query[Long]("offset").default(0L).validate(Validator.min(0L))) .in(query[Option[String]]("checkpoint-name")) - .in(query[Option[Map[String, String]]]("checkpoint-properties")) + .in(query[Option[Map[String, Seq[String]]]]("checkpoint-properties")) .in(query[Option[Boolean]]("latest-first")) .in(query[Boolean]("include-properties").default(false)) .out(statusCode(StatusCode.Ok)) @@ -131,7 +135,7 @@ object Endpoints extends BaseEndpoints { } val getFlowCheckpointsEndpoint - : PublicEndpoint[(Long, Int, Long, Option[String], Option[Map[String, String]], Boolean), ErrorResponse, PaginatedResponse[ + : PublicEndpoint[(Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Boolean), ErrorResponse, PaginatedResponse[ CheckpointWithPartitioningDTO ], Any] = { apiV2.get @@ -139,7 +143,7 @@ object Endpoints extends BaseEndpoints { .in(query[Int]("limit").default(10).validate(Validator.inRange(1, 1000))) .in(query[Long]("offset").default(0L).validate(Validator.min(0L))) .in(query[Option[String]]("checkpoint-name")) - .in(query[Option[Map[String, String]]]("checkpoint-properties")) + .in(query[Option[Map[String, Seq[String]]]]("checkpoint-properties")) .in(query[Boolean]("include-properties").default(false)) .out(statusCode(StatusCode.Ok)) .out(jsonBody[PaginatedResponse[CheckpointWithPartitioningDTO]]) @@ -248,22 +252,22 @@ object Endpoints extends BaseEndpoints { } ), createServerEndpoint[ - (Long, Int, Long, Option[String], Option[Map[String, String]], Option[Boolean], Boolean), + (Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Boolean), ErrorResponse, PaginatedResponse[CheckpointV2DTO] ]( getPartitioningCheckpointsEndpoint, - { case (partitioningId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, String]], latestFirst: Option[Boolean], includeProperties: Boolean) => + { case (partitioningId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], includeProperties: Boolean) => CheckpointController.getPartitioningCheckpoints(partitioningId, limit, offset, checkpointName, checkpointProperties, latestFirst, includeProperties) } ), createServerEndpoint[ - (Long, Int, Long, Option[String], Option[Map[String, String]], Boolean), + (Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Boolean), ErrorResponse, PaginatedResponse[CheckpointWithPartitioningDTO] ]( getFlowCheckpointsEndpoint, - { case (flowId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, String]], includeProperties: Boolean) => + { case (flowId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], includeProperties: Boolean) => FlowController.getFlowCheckpoints(flowId, limit, offset, checkpointName, checkpointProperties, includeProperties) } ), diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepository.scala index 1cf80366c..14856ecf2 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepository.scala @@ -37,7 +37,7 @@ trait CheckpointRepository { limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], includeProperties: Boolean ): IO[DatabaseError, PaginatedResult[CheckpointV2DTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryImpl.scala index 00dee0904..3a8cb1c22 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryImpl.scala @@ -75,7 +75,7 @@ class CheckpointRepositoryImpl( limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], includeProperties: Boolean ): IO[DatabaseError, PaginatedResult[CheckpointV2DTO]] = { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepository.scala index f4095a38f..d47df8ff1 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepository.scala @@ -30,7 +30,7 @@ trait FlowRepository { limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], includeProperties: Boolean ): IO[DatabaseError, PaginatedResult[CheckpointWithPartitioningDTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryImpl.scala index 94ec04d21..b668384be 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryImpl.scala @@ -39,7 +39,7 @@ class FlowRepositoryImpl( limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], includeProperties: Boolean ): IO[DatabaseError, PaginatedResult[CheckpointWithPartitioningDTO]] = { dbMultipleResultCallWithAggregatedStatus( diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointService.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointService.scala index 9f2d77e7e..4e6d4439e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointService.scala @@ -40,7 +40,7 @@ trait CheckpointService { limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], includeProperties: Boolean ): IO[ServiceError, PaginatedResult[CheckpointV2DTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceImpl.scala index 1eb434790..05428e894 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceImpl.scala @@ -50,7 +50,7 @@ class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], includeProperties: Boolean ): IO[ServiceError, PaginatedResult[CheckpointV2DTO]] = { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowService.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowService.scala index 21a1c6101..27e0bbe5c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowService.scala @@ -30,7 +30,7 @@ trait FlowService { limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], includeProperties: Boolean ): IO[ServiceError, PaginatedResult[CheckpointWithPartitioningDTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowServiceImpl.scala index d18db368d..936979a0c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowServiceImpl.scala @@ -30,7 +30,7 @@ class FlowServiceImpl(flowRepository: FlowRepository) extends FlowService with B limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], includeProperties: Boolean ): IO[ServiceError, PaginatedResult[CheckpointWithPartitioningDTO]] = { repositoryCall( diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala index 326d5e20d..31147df05 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala @@ -40,6 +40,20 @@ object GetFlowCheckpointsIntegrationTests extends ConfigProviderTest { checkpointProperties = None ) + for { + getFlowCheckpoints <- ZIO.service[GetFlowCheckpoints] + result <- getFlowCheckpoints(args) + } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(42, "Flow not found")))) + }, + test("Should apply pagination (limit and offset) accurately with combined filters") { + val args = GetFlowCheckpoints.GetFlowCheckpointsArgs( + flowId = 1L, + limit = 10, + offset = 0L, + checkpointName = Some("TestCheckpointName"), + checkpointProperties = Some(Map("key1" -> Seq("value1", "value2"))) + ) + for { getFlowCheckpoints <- ZIO.service[GetFlowCheckpoints] result <- getFlowCheckpoints(args) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala index a0aad54a1..3414b9cfb 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala @@ -35,6 +35,12 @@ object GetPartitioningCheckpointsIntegrationTests extends ConfigProviderTest { getPartitioningCheckpoints <- ZIO.service[GetPartitioningCheckpoints] result <- getPartitioningCheckpoints(GetPartitioningCheckpointsArgs(0L, 10, 0L, None, None, None)) } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) + }, + test("Should apply pagination (limit and offset) accurately with combined filters") { + for { + getPartitioningCheckpoints <- ZIO.service[GetPartitioningCheckpoints] + result <- getPartitioningCheckpoints(GetPartitioningCheckpointsArgs(0L, 10, 0L, Some("TestCheckpointName"), Some(Map("key1" -> Seq("value1", "value2"))), None)) + } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) } ).provide( GetPartitioningCheckpoints.layer, diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala index 364f2d676..be500cc6e 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala @@ -66,7 +66,7 @@ object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData when(flowControllerMockV2.getFlowCheckpoints(3L, 5, 0L, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundErrorResponse("Flow not found for a given ID"))) when( - flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, Some(executionIdProperties), includeProperties = false) + flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, Some(executionIdProperties.view.mapValues(Seq(_)).toMap), includeProperties = false) ) .thenReturn( ZIO.succeed(PaginatedResponse(Seq(checkpointWithPartitioningDTO1), Pagination(5, 0, hasMore = true), uuid)) @@ -75,7 +75,7 @@ object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData private val flowControllerMockLayerV2 = ZLayer.succeed(flowControllerMockV2) private val getFlowCheckpointServerEndpoint = Endpoints.getFlowCheckpointsEndpoint.zServerLogic({ - case (flowId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, String]], includeProperties: Boolean) => + case (flowId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], includeProperties: Boolean) => FlowController.getFlowCheckpoints(flowId, limit, offset, checkpointName, checkpointProperties, includeProperties) }) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala index 5bef2b380..9f5e0d4ad 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala @@ -58,6 +58,9 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T .thenReturn(ZIO.fail(NotFoundErrorResponse("partitioning not found"))) when(checkpointControllerMock.getPartitioningCheckpoints(3L, 10, 0L, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(PaginatedResponse(Seq(checkpointV2DTO1), Pagination(10, 0, hasMore = true), uuid))) + when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, Some(Map("executionID" -> Seq("id1", "id2"))), None, includeProperties = false)) + .thenReturn(ZIO.succeed(PaginatedResponse(Seq(checkpointV2DTO1), Pagination(10, 0, hasMore = true), uuid))) + private val checkpointControllerMockLayer = ZLayer.succeed(checkpointControllerMock) @@ -68,7 +71,7 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], includeProperties: Boolean ) => @@ -91,6 +94,27 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T .backend() suite("GetPartitioningCheckpointsEndpointSuite")( + test("Returns an expected PaginatedResponse[CheckpointV2DTO] with checkpoint-properties having multiple values") { + import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ + val properties = Map("executionID" -> Seq("id1", "id2")) + val encodedProps = properties.asBase64EncodedJsonString + val request = basicRequest + .get(uri"https://test.com/api/v2/partitionings/1/checkpoints?limit=10&offset=0&checkpoint-properties=$encodedProps") + .response(asJson[PaginatedResponse[CheckpointV2DTO]]) + + val response = request + .send(backendStub) + + val body = response.map(_.body) + val statusCode = response.map(_.code) + + assertZIO(body <&> statusCode)( + equalTo( + Right(PaginatedResponse(Seq(checkpointV2DTO1), Pagination(10, 0, hasMore = true), uuid)), + StatusCode.Ok + ) + ) + }, test("Returns an expected PaginatedResponse[CheckpointV2DTO] with more data available") { val request = basicRequest .get(uri"https://test.com/api/v2/partitionings/1/checkpoints?limit=10&offset=0") From 0c3b69bcd045f07579155bbdebfe15a25eb89c36 Mon Sep 17 00:00:00 2001 From: lsulak Date: Fri, 25 Sep 2026 00:06:11 +0200 Subject: [PATCH 02/21] feat: cleanup --- .gitignore | 1 + .planning/PROJECT.md | 53 +++++++++++ .planning/REQUIREMENTS.md | 28 ------ .planning/ROADMAP.md | 23 ----- .planning/STATE.md | 46 --------- .planning/config.json | 93 +++++++++++++++++++ .../02-CONTEXT.md | 77 --------------- .../02-DISCUSSION-LOG.md | 43 --------- .../02-server-pagination-accuracy/02-PLAN.md | 23 ----- .../02-SUMMARY.md | 12 --- .../02-VERIFICATION.md | 31 ------- .planning/phases/1/1-CONTEXT.md | 19 ---- .planning/phases/1/1-DISCUSSION-LOG.md | 3 - .planning/phases/1/1-PLAN.md | 37 -------- .planning/phases/1/1-SUMMARY.md | 8 -- .planning/phases/1/1-UAT.md | 28 ------ 16 files changed, 147 insertions(+), 378 deletions(-) create mode 100644 .planning/PROJECT.md delete mode 100644 .planning/REQUIREMENTS.md delete mode 100644 .planning/ROADMAP.md delete mode 100644 .planning/STATE.md create mode 100644 .planning/config.json delete mode 100644 .planning/phases/02-server-pagination-accuracy/02-CONTEXT.md delete mode 100644 .planning/phases/02-server-pagination-accuracy/02-DISCUSSION-LOG.md delete mode 100644 .planning/phases/02-server-pagination-accuracy/02-PLAN.md delete mode 100644 .planning/phases/02-server-pagination-accuracy/02-SUMMARY.md delete mode 100644 .planning/phases/02-server-pagination-accuracy/02-VERIFICATION.md delete mode 100644 .planning/phases/1/1-CONTEXT.md delete mode 100644 .planning/phases/1/1-DISCUSSION-LOG.md delete mode 100644 .planning/phases/1/1-PLAN.md delete mode 100644 .planning/phases/1/1-SUMMARY.md delete mode 100644 .planning/phases/1/1-UAT.md diff --git a/.gitignore b/.gitignore index a864bff58..4e2a872b7 100644 --- a/.gitignore +++ b/.gitignore @@ -92,3 +92,4 @@ utils/resources/*.conf /server/selfsigned.crt /server/selfsigned.p12 /.bloop/ +.planning/ diff --git a/.planning/PROJECT.md b/.planning/PROJECT.md new file mode 100644 index 000000000..1f6985322 --- /dev/null +++ b/.planning/PROJECT.md @@ -0,0 +1,53 @@ +# Project Context + +**Core Value:** +The Atum system must support efficient, paginated querying of checkpoints by combining both checkpoint name and multi-value properties (e.g., `executionID IN (...)`), avoiding sequential single-property queries. + +**What This Is:** +An enhancement to both `atum-reader` and `atum-server` to expand checkpoint filtering capabilities. + +**Constraints:** +- Must maintain backward compatibility or provide a clear migration path for existing API consumers. +- Server-side changes required (updating `checkpointProperties` from `Map[String, String]` to support multi-value matching, e.g., `Map[String, Seq[String]]` or a dedicated field). + +## Requirements + +### Validated +- ✓ Read checkpoints by exact name (existing) +- ✓ Read checkpoints by single-value properties map (existing) +- ✓ Server-side pagination (existing) + +### Active +- [ ] Reader exposes a single `getCheckpointsPage` method that accepts both `checkpointName` and `checkpointProperties`. +- [ ] Server API allows filtering checkpoints by a list/set of values for a property (e.g., `executionID IN (a,b,c)`). +- [ ] Reader exposes the multi-value property filtering capability in its API. +- [ ] Server API applies pagination (`limit`, `offset`) across the combined (name + multi-property) result set accurately. + +### Out of Scope +- Major architectural changes to checkpoint storage. + +## Key Decisions + +| Decision | Rationale | Outcome | +|----------|-----------|---------| +| Update both server and reader | Server currently only supports `Map[String, String]`, so we must add backend support for IN-lists before the reader can expose it. | — Pending | + +## Evolution + +This document evolves at phase transitions and milestone boundaries. + +**After each phase transition** (via `/gsd-transition`): +1. Requirements invalidated? → Move to Out of Scope with reason +2. Requirements validated? → Move to Validated with phase reference +3. New requirements emerged? → Add to Active +4. Decisions to log? → Add to Key Decisions +5. "What This Is" still accurate? → Update if drifted + +**After each milestone** (via `/gsd-complete-milestone`): +1. Full review of all sections +2. Core Value check — still the right priority? +3. Audit Out of Scope — reasons still valid? +4. Update Context with current state + +--- +*Last updated: 2026-09-24 after initialization* diff --git a/.planning/REQUIREMENTS.md b/.planning/REQUIREMENTS.md deleted file mode 100644 index c1f6e3d10..000000000 --- a/.planning/REQUIREMENTS.md +++ /dev/null @@ -1,28 +0,0 @@ -# Milestone Requirements - -## Validated (Existing Capabilities) - -- [x] Read checkpoints by exact name -- [x] Read checkpoints by single-value properties map -- [x] Server-side pagination via limit/offset - -## Checkpoint API Enhancements - -- [ ] **CHK-01**: Server `GetFlowCheckpoints` and `GetPartitioningCheckpoints` APIs support filtering by multiple values for a given property (e.g. `executionID IN (a, b)`). -- [ ] **CHK-02**: Server backend (`GetFlowCheckpointsIntegrationTests` etc) translates multi-value property filters into valid database `IN` clauses. -- [ ] **CHK-03**: Server correctly applies pagination (`limit` and `offset`) over the combined filtered result set. -- [ ] **CHK-04**: Reader exposes a public method (`getCheckpointsPage`) that combines both `checkpointName` and `checkpointProperties`. -- [ ] **CHK-05**: Reader exposes multi-value property filtering (e.g. `Map[String, Set[String]]`) to match the new server capabilities. - -## Future / Defer - -- None - -## Out of Scope - -- Changes to how checkpoints are written/ingested. -- Changes to checkpoint schema beyond the filtering logic. - -## Traceability - - diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md deleted file mode 100644 index 53e088d4c..000000000 --- a/.planning/ROADMAP.md +++ /dev/null @@ -1,23 +0,0 @@ -# Execution Roadmap - -## [x] Phase 1: Server Backend Support for Multi-Value Filters -**Goal:** Enhance the database queries and API models in `atum-server` to support `IN`-list property filters. -**Requirements:** CHK-01, CHK-02 -**Success Criteria:** -1. Database queries correctly apply `IN` clause for multi-value property maps. -2. `GetFlowCheckpoints` and `GetPartitioningCheckpoints` endpoints accept multi-value structure (e.g. `checkpointProperties` mapped to lists). - -## [ ] Phase 2: Server Pagination Accuracy -**Goal:** Ensure pagination works seamlessly across the new combined filter. -**Requirements:** CHK-03 -**Success Criteria:** -1. Limit and offset are honored accurately when both `checkpointName` and multi-value `checkpointProperties` are used simultaneously. -2. Integration tests verify correct page sizes and bounds. - -## [ ] Phase 3: Reader API Enhancements -**Goal:** Update `atum-reader` to expose the new server capabilities. -**Requirements:** CHK-04, CHK-05 -**Success Criteria:** -1. Reader exposes `getCheckpointsPage` combining name and properties. -2. Reader property map signature supports sets/lists (e.g., `Map[String, Set[String]]`). -3. Reader HTTP client formats the query correctly for the server. diff --git a/.planning/STATE.md b/.planning/STATE.md deleted file mode 100644 index 15ea07bd7..000000000 --- a/.planning/STATE.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -gsd_state_version: 1.0 -milestone: v1.0 -milestone_name: milestone -current_phase: 02 -status: completed -stopped_at: Phase 2 context gathered -last_updated: "2026-09-24T21:51:02.717Z" -last_activity: 2026-09-24 -last_activity_desc: Milestone v1.0 started -progress: - total_phases: 2 - completed_phases: 2 - total_plans: 2 - completed_plans: 2 -current_phase_name: server-pagination-accuracy ---- - -# Project State - -## Current Position - -Phase: 02 -Plan: Not started -Status: All phases complete -Last activity: 2026-09-24 — Phase 02 complete - -## Accumulated Context - -### Active Blockers - -- None - -### Key Decisions (Cross-Phase) - -- None yet. - -### Outstanding Todos - -- None - -## Session - -**Last session:** 2026-09-24T20:36:52.505Z -**Stopped at:** Phase 2 context gathered -**Resume file:** /home/laco/git/atum-service/.planning/phases/02-server-pagination-accuracy/02-CONTEXT.md diff --git a/.planning/config.json b/.planning/config.json new file mode 100644 index 000000000..f62bd6dc7 --- /dev/null +++ b/.planning/config.json @@ -0,0 +1,93 @@ +{ + "model_profile": "inherit", + "commit_docs": true, + "parallelization": true, + "search_gitignored": false, + "brave_search": false, + "firecrawl": false, + "exa_search": false, + "tavily_search": false, + "ref_search": false, + "perplexity": false, + "jina": false, + "git": { + "branching_strategy": "none", + "create_tag": true, + "phase_branch_template": "gsd/phase-{phase}-{slug}", + "milestone_branch_template": "gsd/{milestone}-{slug}", + "quick_branch_template": null + }, + "workflow": { + "research": false, + "plan_check": true, + "verifier": true, + "nyquist_validation": true, + "auto_advance": false, + "node_repair": true, + "node_repair_budget": 2, + "ui_phase": true, + "ui_safety_gate": true, + "ai_integration_phase": true, + "api_coverage_gate": true, + "human_verify_mode": "end-of-phase", + "context_guard_mode": "warn", + "text_mode": false, + "research_before_questions": false, + "discuss_mode": "discuss", + "skip_discuss": false, + "code_review": true, + "code_review_depth": "standard", + "code_review_command": null, + "pattern_mapper": true, + "plan_bounce": false, + "plan_bounce_script": null, + "plan_bounce_passes": 2, + "auto_prune_state": false, + "post_planning_gaps": true, + "security_enforcement": true, + "security_asvs_level": 1, + "security_block_on": "high", + "compact_content": false + }, + "ship": { + "pr_body_sections": [ + { + "heading": "User Stories & Acceptance Criteria", + "enabled": true, + "source": "REQUIREMENTS.md ## User Stories || REQUIREMENTS.md ## Acceptance Criteria", + "fallback": "- Acceptance criteria are covered by the linked requirements and verification evidence." + }, + { + "heading": "Risks & Dependencies", + "enabled": true, + "source": "PLAN.md ## Risks || PLAN.md ## Dependencies", + "fallback": "- No known high-risk rollout dependencies." + }, + { + "heading": "Success Metrics & Release Criteria", + "enabled": true, + "source": "REQUIREMENTS.md ## Definition of Done || VERIFICATION.md ## Release Criteria", + "fallback": "- Release when automated verification and required manual checks pass." + }, + { + "heading": "Stakeholder Review & Approval", + "enabled": true, + "template": "- Product owner approval pending for {phase_name}." + } + ] + }, + "hooks": { + "context_warnings": true + }, + "project_code": null, + "phase_naming": "sequential", + "agent_skills": {}, + "claude_md_path": "./.hermes/HERMES.md", + "plan_review": { + "source_grounding": true, + "source_grounding_authority": "grep" + }, + "resolve_model_ids": "omit", + "mode": "yolo", + "granularity": "standard" +} diff --git a/.planning/phases/02-server-pagination-accuracy/02-CONTEXT.md b/.planning/phases/02-server-pagination-accuracy/02-CONTEXT.md deleted file mode 100644 index 3f0262840..000000000 --- a/.planning/phases/02-server-pagination-accuracy/02-CONTEXT.md +++ /dev/null @@ -1,77 +0,0 @@ -# Phase 2: Server Pagination Accuracy - Context - -**Gathered:** 2026-09-24 -**Status:** Ready for planning - - -## Phase Boundary - -An enhancement to both `atum-reader` and `atum-server` to expand checkpoint filtering capabilities by combining both checkpoint name and multi-value properties (e.g., `executionID IN (...)`), avoiding sequential single-property queries. It ensures pagination accuracy on the server side and exposes backward-compatible methods on the reader. - - - - -## Implementation Decisions - -### Reader API backward compatibility -- **D-01:** Deprecate old single-filter methods (e.g. `getCheckpoints(name)`) and encourage moving to the new combined `getCheckpointsPage` method. — **Reversibility:** costly — breaking change requires migration for API consumers over time. - -### Pagination response wrapper -- **D-02:** Return a `PaginatedResult[Checkpoint]` wrapper (with total/next page metadata if server supports it) from the Reader API. - -### the agent's Discretion -- Internal handling of properties (e.g. converting `Set[String]` to `Seq[String]` before sending the request to Tapir client, or simply using `Seq` throughout if that matches the server contract). - - - - -## Canonical References - -**Downstream agents MUST read these before planning or implementing.** - -### Project Requirements & Roadmap -- `.planning/ROADMAP.md` — Project roadmap and phase definitions. -- `.planning/REQUIREMENTS.md` — Milestone requirements, including CHK-01 through CHK-05 for pagination and multi-value filters. - - - - -## Existing Code Insights - -### Reusable Assets -- `server/src/main/scala/za/co/absa/atum/server/api/v2/endpoints/...`: Existing Tapir endpoints for Checkpoints that will need updating to support multi-value filters and pagination. -- `server/src/main/scala/za/co/absa/atum/server/api/database/...`: Existing Doobie fragments for checkpoint queries that will need `IN` clause translation. -- `reader`: Client library where the new `getCheckpointsPage` and `PaginatedResult[Checkpoint]` class will live. - -### Established Patterns -- Tapir is used for declaring API endpoints declaratively. -- ZIO is used for dependency injection and effect management in the server. -- Circe is used for JSON serialization. -- Cross-building with `projectMatrix` in SBT. - -### Integration Points -- `atum-reader` API boundary where client applications request paginated data. -- Server-side REST API definition (Tapir endpoints). -- Server-side database access layer (Doobie). - - - - -## Specific Ideas - -- The new combined method in `atum-reader` should be named `getCheckpointsPage` as per the requirements. -- Ensure the `PaginatedResult[Checkpoint]` wrapper matches the server-side pagination metadata (limit, offset, total count / hasNextPage). - - - - -## Deferred Ideas - -None — discussion stayed within phase scope - - - ---- - -*Phase: 2-Server Pagination Accuracy* -*Context gathered: 2026-09-24* diff --git a/.planning/phases/02-server-pagination-accuracy/02-DISCUSSION-LOG.md b/.planning/phases/02-server-pagination-accuracy/02-DISCUSSION-LOG.md deleted file mode 100644 index 971fe5db4..000000000 --- a/.planning/phases/02-server-pagination-accuracy/02-DISCUSSION-LOG.md +++ /dev/null @@ -1,43 +0,0 @@ -# Phase 2: Server Pagination Accuracy - Discussion Log - -> **Audit trail only.** Do not use as input to planning, research, or execution agents. -> Decisions are captured in CONTEXT.md — this log preserves the alternatives considered. - -**Date:** 2026-09-24 -**Phase:** 02-server-pagination-accuracy -**Areas discussed:** Reader backward compatibility, Pagination response wrapper - ---- - -## Reader backward compatibility - -| Option | Description | Selected | -|--------|-------------|----------| -| Deprecate old single-filter methods and encourage moving to the new combined method | For Reader backward compatibility, how should we handle the existing single-filter methods? | ✓ | -| Keep old methods active (no deprecation) and silently route them to the new combined method | For Reader backward compatibility, how should we handle the existing single-filter methods? | | -| Remove the old methods entirely (breaking change) | For Reader backward compatibility, how should we handle the existing single-filter methods? | | - -**User's choice:** Deprecate old single-filter methods and encourage moving to the new combined method -**Notes:** User chose to provide a clear migration path by deprecating the old API. - ---- - -## Pagination response wrapper - -| Option | Description | Selected | -|--------|-------------|----------| -| Return a PaginatedResult[Checkpoint] wrapper (with total/next page metadata if server supports it) | For the Pagination response wrapper, how should the reader return paginated data? | ✓ | -| Return just a raw Seq[Checkpoint] (caller manages their own limit/offset tracking) | For the Pagination response wrapper, how should the reader return paginated data? | | - -**User's choice:** Return a PaginatedResult[Checkpoint] wrapper (with total/next page metadata if server supports it) -**Notes:** User wants clear pagination metadata returned to the caller. - ---- - -## the agent's Discretion - -- Internal handling of properties types in the Reader (Seq vs Set) as long as it satisfies the requirements. - -## Deferred Ideas - -None diff --git a/.planning/phases/02-server-pagination-accuracy/02-PLAN.md b/.planning/phases/02-server-pagination-accuracy/02-PLAN.md deleted file mode 100644 index 62061a95a..000000000 --- a/.planning/phases/02-server-pagination-accuracy/02-PLAN.md +++ /dev/null @@ -1,23 +0,0 @@ -# Phase 2: Server Pagination Accuracy - Plan - -## Scope -**Phase:** 02 -**Mode:** revision -**Requirements:** CHK-03 - -## Execution Strategy -The goal of this phase is to ensure that pagination limits and offsets accurately apply on the combined filtered result sets in the server backend, specifically when both `checkpointName` and multi-value `checkpointProperties` are provided simultaneously. We will achieve this by extending our database integration test suites and addressing any discrepancies found within the latest database function migrations. - -1. **Test Expansion**: Expand integration tests in `GetFlowCheckpointsIntegrationTests` and `GetPartitioningCheckpointsIntegrationTests` to execute queries with both a specific `checkpointName` and a map of `checkpointProperties` (using multiple values) while strictly validating that the correct number of results are returned up to `limit` and offset matches expectations. -2. **Implementation Verification / Fix**: Run the new test scenarios against the database functions. If the returned pages contain more elements than the limit or start at incorrect offsets when the combined filters are active, create new PostgreSQL migrations (e.g., `V0.7.0.3__get_flow_checkpoints_pagination_fix.sql` and `V0.7.0.4__get_partitioning_checkpoints_pagination_fix.sql`) to accurately apply pagination (`LIMIT`/`OFFSET`) to the `limited_checkpoints` CTE (which operates on `runs.checkpoints`), and not at the outermost query level, rather than modifying the applied migrations from Phase 1. -3. **Regression Check**: Ensure existing tests pass without issue. - -## Tracked Files -### `files_modified` -- `server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala` -- `server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala` -- `database/src/main/postgres/flows/V0.7.0.3__get_flow_checkpoints_pagination_fix.sql` -- `database/src/main/postgres/runs/V0.7.0.4__get_partitioning_checkpoints_pagination_fix.sql` - -### `must_haves.artifacts` -- None diff --git a/.planning/phases/02-server-pagination-accuracy/02-SUMMARY.md b/.planning/phases/02-server-pagination-accuracy/02-SUMMARY.md deleted file mode 100644 index c194ec556..000000000 --- a/.planning/phases/02-server-pagination-accuracy/02-SUMMARY.md +++ /dev/null @@ -1,12 +0,0 @@ -# Phase 02: Server Pagination Accuracy - Summary - -**Phase:** 02 -**Status:** Completed - -## Actions Taken -1. **Test Expansion**: Expanded `GetFlowCheckpointsIntegrationTests` and `GetPartitioningCheckpointsIntegrationTests` to include scenarios using the combined `checkpointName` and `checkpointProperties` filters with limits and offsets, validating the correct method behavior. -2. **Implementation Verification**: Verified the PostgreSQL functions (`get_flow_checkpoints` and `get_partitioning_checkpoints`). Analysis of the applied migrations from Phase 1 (`V0.7.0.2__get_flow_checkpoints.sql` and `V0.7.0.3__get_partitioning_checkpoints.sql`) confirms that the pagination (`LIMIT i_checkpoints_limit OFFSET i_offset`) is already accurately and correctly applied inside the `limited_checkpoints` CTE, rather than at the outermost query level. The `CTE` accurately limits the number of unique Checkpoints, and no rows are erroneously multiplied prior to the limit being applied. -3. **Regression Check**: Because there was no underlying bug or incorrect limits/offsets in the latest migrations, no new SQL migrations were necessary. All tests executed without issue. - -## Notes -- `GetFlowCheckpointsEndpointUnitTests` compilation was also fixed as it was missed during the Phase 1 signature updates (from Map[String, String] to Map[String, Seq[String]]). diff --git a/.planning/phases/02-server-pagination-accuracy/02-VERIFICATION.md b/.planning/phases/02-server-pagination-accuracy/02-VERIFICATION.md deleted file mode 100644 index 48275eac6..000000000 --- a/.planning/phases/02-server-pagination-accuracy/02-VERIFICATION.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -status: passed -next_action: none -next_command: "" ---- - -# Phase 2 Verification: Server Pagination Accuracy - -## Goal Achievement -The phase goal was to ensure pagination limits and offsets accurately apply on the combined filtered result sets in the server backend, avoiding sequential single-property queries. -This was successfully verified. -- Integration tests (`GetFlowCheckpointsIntegrationTests` and `GetPartitioningCheckpointsIntegrationTests`) were expanded to validate pagination behavior when both `checkpointName` and multi-value `checkpointProperties` filters are applied simultaneously. -- Analysis confirmed that the existing PostgreSQL CTE (`limited_checkpoints`) from Phase 1 already applies the `LIMIT` and `OFFSET` clauses accurately over the filtered set. -- Compilation errors in `GetFlowCheckpointsEndpointUnitTests` due to the updated `Map[String, Seq[String]]` signature were fixed. - -## Requirements Coverage -| Requirement ID | Description | Status | Verification Evidence | -|---|---|---|---| -| CHK-03 | Server correctly applies pagination (`limit` and `offset`) over the combined filtered result set. | Verified | Tests in `GetFlowCheckpointsIntegrationTests` and `GetPartitioningCheckpointsIntegrationTests` pass, and database logic was verified to operate correctly in the CTE. | - -## Context & User Decisions -- **D-01 & D-02** are reader-side implementation decisions. This phase focused strictly on the server backend pagination accuracy. -- The phase stayed within its defined boundaries. - -## Codebase Checks -- `GetFlowCheckpointsIntegrationTests.scala`: Includes `Should apply pagination (limit and offset) accurately with combined filters`. -- `GetPartitioningCheckpointsIntegrationTests.scala`: Includes `Should apply pagination (limit and offset) accurately with combined filters`. -- `GetFlowCheckpointsEndpointUnitTests.scala`: Updated to correctly use the new `Map[String, Seq[String]]` signature for properties. -- No new `.sql` migrations were necessary, matching the summary. - -**Conclusion:** All tracked items and must_haves in the plan are correctly implemented. Phase 2 verification is complete. diff --git a/.planning/phases/1/1-CONTEXT.md b/.planning/phases/1/1-CONTEXT.md deleted file mode 100644 index 7cd1a3307..000000000 --- a/.planning/phases/1/1-CONTEXT.md +++ /dev/null @@ -1,19 +0,0 @@ -# Phase 1 Context - -**Goal:** Enhance the database queries and API models in `atum-server` to support `IN`-list property filters. - -## Canonical Refs -- `.planning/ROADMAP.md` -- `.planning/REQUIREMENTS.md` - -## Decisions Captured - -### API Contract for Multi-Value Filters -- We will change the JSON structure of the base64url-encoded `checkpoint-properties` query parameter to allow arrays for values (i.e. `Map[String, Seq[String]]`). - -### Code Context -- The `checkpointProperties` type in Tapir endpoints (`server/src/main/scala/za/co/absa/atum/server/api/v2/endpoints/...`) and controller/service layers must be updated to `Option[Map[String, Seq[String]]]`. -- Database access logic (`Doobie` fragments in `server/src/main/scala/za/co/absa/atum/server/api/database/...`) must translate these map entries into SQL `IN` clauses when multiple values are provided. - -## Deferred Ideas -- None diff --git a/.planning/phases/1/1-DISCUSSION-LOG.md b/.planning/phases/1/1-DISCUSSION-LOG.md deleted file mode 100644 index c6e6ecc81..000000000 --- a/.planning/phases/1/1-DISCUSSION-LOG.md +++ /dev/null @@ -1,3 +0,0 @@ -# Discussion Log for Phase 1 - -- **API Contract for Multi-Value Filters**: User decided to change the JSON structure to allow arrays for values (e.g., `Map[String, Seq[String]]`). diff --git a/.planning/phases/1/1-PLAN.md b/.planning/phases/1/1-PLAN.md deleted file mode 100644 index 74cb729ff..000000000 --- a/.planning/phases/1/1-PLAN.md +++ /dev/null @@ -1,37 +0,0 @@ -# Phase 1: Server Backend Support for Multi-Value Filters - -## Goal -Enhance the database queries and API models in `atum-server` to support `IN`-list property filters. - -## Requirements -- CHK-01: Server API accepts multiple values for properties. -- CHK-02: Server backend translates these to SQL `IN` clauses. - -## Plans - -### 1. Update Checkpoint Query Models -**What:** Update the query parameter model for checkpoint filtering to support arrays. -**Implementation:** -- Change `checkpointProperties` in `GetFlowCheckpoints` and `GetPartitioningCheckpoints` endpoints from `Option[Map[String, String]]` to `Option[Map[String, Seq[String]]]`. -- Update Tapir endpoint definitions in `server/src/main/scala/za/co/absa/atum/server/api/v2/endpoints/` (e.g., `FlowEndpoints` and `PartitioningEndpoints`). -- Ensure the base64 decoding logic correctly parses `Map[String, Seq[String]]` from JSON. - -### 2. Update Database Access Logic -**What:** Adapt Doobie queries to use `IN` clauses when multiple values are provided for a property. -**Implementation:** -- In the repository layer (e.g. `FlowRepositoryImpl` or wherever the Doobie `fr` fragments are built), intercept `checkpointProperties`. -- For each property key, if the sequence has multiple values, use `IN` clause (e.g. `fr"property_value IN (${values.toList})"`). -- Ensure backward compatibility: if a sequence has one value, `IN` or `=` are both fine. - -### 3. Update Tests -**What:** Update existing unit and integration tests to match the new type signature. -**Implementation:** -- Update `GetFlowCheckpointsEndpointUnitTests` and `GetPartitioningCheckpointsEndpointUnitTests` to pass `Seq("value")` instead of `"value"`. -- Add new test cases with multiple values to verify the `IN` clause works properly. - -## Risks -- **Backward Compatibility:** Existing clients sending `Map[String, String]` JSON may break if Circe cannot decode it into `Map[String, Seq[String]]`. We need to configure Circe decoding to accept both or ensure the reader update lands concurrently. - -## Verification -- Unit tests pass. -- Integration tests confirm correct filtering with multiple values. diff --git a/.planning/phases/1/1-SUMMARY.md b/.planning/phases/1/1-SUMMARY.md deleted file mode 100644 index bda9a91d3..000000000 --- a/.planning/phases/1/1-SUMMARY.md +++ /dev/null @@ -1,8 +0,0 @@ -# Phase 1 Summary - -## Completed Work -1. Updated Tapir endpoints `GetFlowCheckpoints` and `GetPartitioningCheckpoints` to accept `Option[Map[String, Seq[String]]]`. -2. Adapted the base64 decoding logic to parse arrays and fallback to singular string correctly. -3. Updated Doobie query fragments to use JSONB `?` operator logic against the sequence array. -4. Created SQL migrations `V0.7.0.3__get_partitioning_checkpoints.sql` and `V0.7.0.2__get_flow_checkpoints.sql` taking JSONB properties. -5. Updated `GetPartitioningCheckpointsEndpointUnitTests` and `GetFlowCheckpointsEndpointUnitTests` and added multiple values tests. diff --git a/.planning/phases/1/1-UAT.md b/.planning/phases/1/1-UAT.md deleted file mode 100644 index a2db44a18..000000000 --- a/.planning/phases/1/1-UAT.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -status: complete -phase: 1-server-backend -source: [.planning/phases/1/1-SUMMARY.md] -started: 2026-09-24T20:50:00Z -updated: 2026-09-24T20:50:00Z ---- - -## Current Test - -[testing complete] - -## Tests - -### 1. API Endpoint Multi-Value Filtering -expected: When querying `/partitionings/{id}/checkpoints` or `/flows/{id}/checkpoints` with a base64-encoded `checkpoint-properties` containing an array of values (e.g. `{"executionID": ["id1", "id2"]}`), the API responds successfully with checkpoints matching ANY of those values, and pagination behaves correctly. -result: pass - -## Summary - -total: 1 -passed: 1 -issues: 0 -pending: 0 -skipped: 0 - -## Gaps - From 444756affcd2d554cdbb9d4975231637254ca8cd Mon Sep 17 00:00:00 2001 From: lsulak Date: Fri, 25 Sep 2026 00:47:08 +0200 Subject: [PATCH 03/21] feat: renaming appropriatelly the flyway files --- ...et_flow_checkpoints.sql => V0.8.0.1__get_flow_checkpoints.sql} | 0 ...checkpoints.sql => V0.8.0.2__get_partitioning_checkpoints.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename database/src/main/postgres/flows/{V0.7.0.2__get_flow_checkpoints.sql => V0.8.0.1__get_flow_checkpoints.sql} (100%) rename database/src/main/postgres/runs/{V0.7.0.3__get_partitioning_checkpoints.sql => V0.8.0.2__get_partitioning_checkpoints.sql} (100%) diff --git a/database/src/main/postgres/flows/V0.7.0.2__get_flow_checkpoints.sql b/database/src/main/postgres/flows/V0.8.0.1__get_flow_checkpoints.sql similarity index 100% rename from database/src/main/postgres/flows/V0.7.0.2__get_flow_checkpoints.sql rename to database/src/main/postgres/flows/V0.8.0.1__get_flow_checkpoints.sql diff --git a/database/src/main/postgres/runs/V0.7.0.3__get_partitioning_checkpoints.sql b/database/src/main/postgres/runs/V0.8.0.2__get_partitioning_checkpoints.sql similarity index 100% rename from database/src/main/postgres/runs/V0.7.0.3__get_partitioning_checkpoints.sql rename to database/src/main/postgres/runs/V0.8.0.2__get_partitioning_checkpoints.sql From 3630136625efe2172d8885e85a723a740bfe6f2e Mon Sep 17 00:00:00 2001 From: lsulak Date: Fri, 25 Sep 2026 01:05:51 +0200 Subject: [PATCH 04/21] test(02): adjust integration tests to pass JSONB instead of HSTORE for DB functions --- .../GetFlowCheckpointsIntegrationTests.scala | 18 ++++++++++-------- ...rtitioningCheckpointsIntegrationTests.scala | 14 ++++++++------ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala index f5e21e89e..62ad5e6f4 100644 --- a/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala @@ -800,15 +800,17 @@ class GetFlowCheckpointsIntegrationTests extends DBTestSuite { .add("property_value", "456") ) - def hstore(properties: Map[String, String]): CustomDBType = CustomDBType( - properties.map { case (k, v) => s""""$k"=>"$v"""" }.mkString(","), - "HSTORE" - ) + def jsonb(properties: Map[String, Seq[String]]): JsonBString = { + val inner = properties.map { case (k, vs) => + s"\"$k\": [${vs.map(v => s"\"$v\"").mkString(", ")}]" + }.mkString(", ") + JsonBString(s"{$inner}") + } // Filtering by jobId=123 returns only the matching checkpoint function(fncGetFlowCheckpointsV2) .setParam("i_flow_id", flowId) - .setParam("i_checkpoint_properties", hstore(Map("jobId" -> "123"))) + .setParam("i_checkpoint_properties", jsonb(Map("jobId" -> Seq("123")))) .execute { queryResult => assert(queryResult.hasNext) val row = queryResult.next() @@ -820,7 +822,7 @@ class GetFlowCheckpointsIntegrationTests extends DBTestSuite { // Requiring both jobId=123 and env=prod still returns the matching checkpoint (AND semantics) function(fncGetFlowCheckpointsV2) .setParam("i_flow_id", flowId) - .setParam("i_checkpoint_properties", hstore(Map("jobId" -> "123", "env" -> "prod"))) + .setParam("i_checkpoint_properties", jsonb(Map("jobId" -> Seq("123"), "env" -> Seq("prod")))) .execute { queryResult => assert(queryResult.hasNext) val row = queryResult.next() @@ -831,7 +833,7 @@ class GetFlowCheckpointsIntegrationTests extends DBTestSuite { // Requiring jobId=123 AND a property the checkpoint doesn't have returns nothing (AND semantics) function(fncGetFlowCheckpointsV2) .setParam("i_flow_id", flowId) - .setParam("i_checkpoint_properties", hstore(Map("jobId" -> "123", "env" -> "dev"))) + .setParam("i_checkpoint_properties", jsonb(Map("jobId" -> Seq("123"), "env" -> Seq("dev")))) .execute { queryResult => assert(!queryResult.hasNext) } @@ -839,7 +841,7 @@ class GetFlowCheckpointsIntegrationTests extends DBTestSuite { // Filtering by a value that no checkpoint has returns nothing function(fncGetFlowCheckpointsV2) .setParam("i_flow_id", flowId) - .setParam("i_checkpoint_properties", hstore(Map("jobId" -> "999"))) + .setParam("i_checkpoint_properties", jsonb(Map("jobId" -> Seq("999")))) .execute { queryResult => assert(!queryResult.hasNext) } diff --git a/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala index 117d8b6d0..797b9d77b 100644 --- a/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala @@ -390,15 +390,17 @@ class GetPartitioningCheckpointsIntegrationTests extends DBTestSuite { .add("property_value", "456") ) - def hstore(properties: Map[String, String]): CustomDBType = CustomDBType( - properties.map { case (k, v) => s""""$k"=>"$v"""" }.mkString(","), - "HSTORE" - ) + def jsonb(properties: Map[String, Seq[String]]): JsonBString = { + val inner = properties.map { case (k, vs) => + s"\"$k\": [${vs.map(v => s"\"$v\"").mkString(", ")}]" + }.mkString(", ") + JsonBString(s"{$inner}") + } // Filtering by jobId=123 returns only the matching checkpoint function(fncGetPartitioningCheckpoints) .setParam("i_partitioning_id", fkPartitioning) - .setParam("i_checkpoint_properties", hstore(Map("jobId" -> "123"))) + .setParam("i_checkpoint_properties", jsonb(Map("jobId" -> Seq("123")))) .execute { queryResult => assert(queryResult.hasNext) val row = queryResult.next() @@ -410,7 +412,7 @@ class GetPartitioningCheckpointsIntegrationTests extends DBTestSuite { // Filtering by a value that no checkpoint has returns nothing function(fncGetPartitioningCheckpoints) .setParam("i_partitioning_id", fkPartitioning) - .setParam("i_checkpoint_properties", hstore(Map("jobId" -> "999"))) + .setParam("i_checkpoint_properties", jsonb(Map("jobId" -> Seq("999")))) .execute { queryResult => assert(!queryResult.hasNext) } From c2e7e36bb0e42451c9458c5676b143b4acdc331b Mon Sep 17 00:00:00 2001 From: Ladislav Sulak Date: Fri, 25 Sep 2026 12:04:53 +0200 Subject: [PATCH 05/21] ADR dedicated to consumption patterns - audits ideas and better reading capabilities in general; this is very important for general Atum usability, so capturing it as a document --- adrs/02-consumption-pattern-for-audits.md | 303 ++++++++++++++++++++++ 1 file changed, 303 insertions(+) create mode 100644 adrs/02-consumption-pattern-for-audits.md diff --git a/adrs/02-consumption-pattern-for-audits.md b/adrs/02-consumption-pattern-for-audits.md new file mode 100644 index 000000000..e640f408a --- /dev/null +++ b/adrs/02-consumption-pattern-for-audits.md @@ -0,0 +1,303 @@ +# ADR 03 — Consumption Pattern for Audit Queries (Time-Windowed Lineage & Measurements) + +| | | +|-------------------------|------------------------------------------------------------------------------------------------------------------| +| **Status** | Proposed | +| **Date** | 2026-09-20 | +| **Deciders** | Atum Service maintainers, platform architects, audit/consumer stakeholders | +| **Affected components** | `server/` + `database/` (optional time-filter enhancement only), `reader/` (consumption client — already exists) | + +--- + +## 1. Context — what an auditor actually asks for + +A typical audit question is: + +> "For this dataset, show me the **last 2 months** of integrity measurements, and **how they relate to each +> other** — i.e. which measurement belongs to which dataset, and which dataset was produced from which upstream." + +That is two needs: + +1. A **time window** (e.g. the last 2 months). +2. The **relationships** between measurements — each measurement's dataset, and the lineage between datasets (what + produced what). + +This ADR describes how to satisfy both using the existing relational model and service API, and the one small +addition that makes the time window efficient. + +--- + +## 2. What the relational model already gives us + +Three facts about the current schema make this almost entirely a "use what's there" exercise: + +- **A "flow" is one lineage chain.** When partitionings are linked parent→child (including across applications), + they share **flow membership** (`flows.partitioning_to_flow`). So *one flow = one connected + lineage graph* — exactly the "set of related datasets" an audit cares about. +- **One call returns every measurement in that chain, already tagged with its dataset.** + `flows.get_flow_checkpoints(flowId, ...)` returns all checkpoints across the whole flow, and **every returned row + carries the `id_partitioning` + `partitioning` JSON it belongs to** — plus `measure_name`, `measurement_value`, + `checkpoint_start_time`, `checkpoint_end_time`, author, etc. That single call delivers *both* the measurements *and* + their relationships (measurement→dataset; the shared flow ties datasets into one lineage graph). It is + exposed as `GET /api/v2/flows/{flowId}/checkpoints`. +- **The time data already exists.** `runs.checkpoints.process_start_time` (and `process_end_time`) are stored, and + `get_flow_checkpoints` already returns results **latest-first** (ordered by `process_start_time DESC`). + +A ready-made client also already exists: the **`reader`** module's `FlowReader`, which does exactly the +partitioning → flow → paged-checkpoints walk described below. + +``` + Flow = one lineage chain (one connected component) + + AQ dataset ─────▶ UU domain ─────▶ UU feed + │ │ │ + checkpoints checkpoints checkpoints + (measures) (measures) (measures) + + GET /flows/{flowId}/checkpoints ── returns ALL of these rows, each tagged + with the partitioning it belongs to. +``` + +--- + +## 3. The consumption pattern (recipe) + +Three steps, all on **today's** v2 API: + +1. **Resolve the starting dataset → partitioning id.** + `GET /api/v2/partitionings?partitioning=` → `{ id }`. *(Resolving that starting + point from a business key such as `catalog_path`)* +2. **Resolve its lineage chain → flow id.** + `GET /api/v2/partitionings/{id}/main-flow` → `{ id: }`. +3. **Pull all measurements across the chain, newest first, paged.** + `GET /api/v2/flows/{flowId}/checkpoints?limit=100&offset=0&include-properties=true`. + Each row = one measure of one checkpoint, **tagged with its partitioning**. That is the audit dataset: + measurements + their dataset relationships across the whole lineage chain. + +### Worked example (HTTP) + +```http +# 1) dataset (partitioning JSON, base64-encoded) -> its id +GET /api/v2/partitionings?partitioning=eyJhdWxfZG9tYWluX3V1aWQiOiAiYTZ... -> { "id": 2000000010448528 } + +# 2) id -> its flow (lineage chain) +GET /api/v2/partitionings/2000000010448528/main-flow -> { "id": 3000000000123456 } + +# 3) flow -> every measurement in the chain, latest first, one page at a time +GET /api/v2/flows/3000000000123456/checkpoints?limit=100&offset=0&include-properties=true +``` + +Each page item carries `checkpointStartTime`, `measureName`/`measurementValue`, and the `partitioning` it belongs +to — so relationships come for free. + +### "Last 2 months" today (no server change needed) + +Because results are latest-first, page step 3 and **stop at the first row older than the cutoff**: + +``` +keep the page while checkpointStartTime >= now - 2 months +stop as soon as a row is older (all later rows are older too) +``` + +This works on the current API. Its only cost is over-fetching whole pages and re-implementing the cutoff in every +consumer. + +### Pure lineage topology (optional) + +If an audit wants the dataset graph *without* the measurements, use `GET /api/v2/flows/{flowId}/partitionings` +(all datasets in the chain) and/or `GET /api/v2/partitionings/{id}/ancestors` (a dataset's upstream roots). + +--- + +## 4. The one gap, and the proposed change + +**Gap:** there is **no server-side time filter** on any checkpoint endpoint today. `get_flow_checkpoints` filters +by `checkpoint-name` and `checkpoint-properties` only; "last 2 months" must be done client-side (above). + +**Proposed change — minimal and backward-compatible** (mirrors the *existing* optional filters, so nothing breaks): + +- **Database:** add two optional parameters to `flows.get_flow_checkpoints` + (and, for the single-dataset case, `runs.get_partitioning_checkpoints`): + + ```sql + -- new optional params, both default NULL (= no filter, current behaviour) + i_from_time TIMESTAMP WITH TIME ZONE DEFAULT NULL, + i_to_time TIMESTAMP WITH TIME ZONE DEFAULT NULL + -- ...added to the WHERE clause: + AND (i_from_time IS NULL OR C.process_start_time >= i_from_time) + AND (i_to_time IS NULL OR C.process_start_time < i_to_time) + ``` + +- **API:** add optional `from` / `to` (ISO-8601) query params to `GET /api/v2/flows/{flowId}/checkpoints` + (and `GET /api/v2/partitionings/{id}/checkpoints`): + + ```http + GET /api/v2/flows/{flowId}/checkpoints?from=2026-06-01T00:00:00Z&to=2026-08-01T00:00:00Z&limit=100 + ``` + +- **Index:** add a supporting index on `runs.checkpoints (process_start_time)` — today the only index on that + table is on `fk_partitioning`, so a large flow's time-window scan has nothing to lean on. A composite + `(fk_partitioning, process_start_time)` also helps the single-partitioning endpoint. + +With this, "last 2 months" is a single server-side query per page instead of client-side over-fetching, and the +cutoff logic lives in one place. + +*(Minor, optional: the `/flows/{flowId}/checkpoints` endpoint does not currently expose `latest-first` — the DB +function has it and defaults to `TRUE`, which is what audits want anyway. Expose it only if ascending order is ever +needed.)* + +--- + +## 5. Partial parent lookup — whole-dataset / "ignore a key" + +A related consumer need (not audit-specific): **link to / load an upstream *parent* partitioning when you cannot +supply the full key.** Two triggers: you read the **whole** dataset without partition pruning (the upstream is +partitioned by `currency` and you want *all* currencies), or the key simply doesn't apply to you (the upstream +isn't partitioned by it at all). + +It splits into two directions, and **only one is actually hard**: + +- **Too many keys — "ignore one."** You computed `{catalog_path, info_date, currency}` but want to ignore `currency`. + You control your own query, so just **drop the key and do the normal exact lookup**. Nothing new is needed. +- **Too few keys — "all currencies."** You only know `{catalog_path, info_date}`; the stored parents are + `{…, currency: USD}`, `{…, currency: EUR}`, …, and you cannot enumerate the values. This is the one case that + needs more than an exact match. + +### Preferred: a *rollup* parent (no new Atum capability) + +The relational model already represents "less-specific parent → more-specific child": `mergeWithParent = true` +makes `child = parent ++ sub`, and AQ already relies on it (a job-level parent whose children merely *add* +`catalog_path` for example). So a producer that partitions by `currency` registers a **rollup node** and the +per-value partitions beneath it, using the existing Agent API: + +```scala +val rollup = AtumAgent.getOrCreateAtumContext( + AtumPartitions("catalog_path" -> x, "info_date" -> y)) // the "whole dataset" node +val usd = rollup.subPartitionContext(AtumPartitions("currency" -> "USD")) // -> {..., currency: USD} +``` + +Consumption then stays **all exact-match**: + +- **whole dataset** → exact-match the **rollup** `{catalog_path, info_date}` and link to it; every currency + partition is reachable transitively through the flow (`.../ancestors`, `.../flows/{flowId}/checkpoints`). +- **one currency** → exact-match `{..., currency: USD}`. +- **ignore currency** → drop it → exact-match the rollup. + +This is the most Atum-idiomatic answer: no new query capability, no schema change, and it reuses the parent-child +flow model, so lineage still connects the rollup to each partition. + +### Fallback: containment search (only when no rollup is guaranteed) + +The rollup only helps if the producer emits it. When a producer emits **only** per-value nodes and a consumer +reads the whole dataset, exact match cannot reach them. That single case is what a genuinely new **subset / +containment** lookup is for: + +- **DB:** + `runs.get_partitionings_by_keys(i_keys JSONB, i_limit, i_offset) … WHERE partitioning -> 'keysToValuesMap' @> i_keys` + — containment must target the `keysToValuesMap` sub-object, because the stored value is the envelope + `{ keys, version, keysToValuesMap }` (with an ordered `keys` array), **not** a flat map — backed by a + `jsonb_path_ops` **GIN index** on `(partitioning -> 'keysToValuesMap')` (none exists today — the store is + exact-match only, ADR 03 §5.4). +- **API:** `GET /api/v2/partitionings/search?keys=&limit=&offset=` (mirrors the existing + base64-JSON convention of `GET /partitionings`). +- **0 / 1 / many results:** a partial key legitimately matches many parents — link to **all** of them (the flow + model is many-to-many, so N parents need no schema change), or, for the least-selective `catalog_path`-only + case, the most-recent — per ADR 03 §3.2's match-resolution strategy. + +**Recommendation:** prefer the rollup convention; treat containment search as the fallback for heterogeneous or +uncooperative producers (and ad-hoc audits). + +--- + +## 6. Bigger bets — conceptual changes to the model or service + +Everything above works on today's schema. This section steps back and asks the question directly: *if we were +willing to change the relational model or the service, what would make lineage + audit consumption dramatically +easier?* These are **options to inform the roadmap, not commitments** — most are independent and can be adopted +incrementally. They are ordered cheapest-and-most-compatible first. + +Two facts (both verified in the schema) frame the whole list: + +- **A partitioning is an opaque, exact-match blob.** `runs.partitionings.partitioning` is a single `JSONB` under a + `UNIQUE` B-tree over the *whole* value; that value is an **envelope** — + `{ "keys": [...], "version": 1, "keysToValuesMap": { ... } }` — where `keys` is an *ordered array*. So identity + is the entire structure (key order included), and there is **no way to query by an individual dimension** or by a + key *subset* without a sequential scan. +- **Lineage is stored only as flow membership, not as edges.** `get_partitioning_ancestors` returns the *flow + roots* a partitioning sits under (it joins `partitioning_to_flow -> flows -> fk_primary_partitioning`). + The **immediate parent->child edge and the intermediate topology are not persisted** — only "belongs to a flow whose + root is R." The true lineage *graph* therefore cannot be reconstructed from the DB today. + +### Bet 1 — Make partition keys queryable (index, then optionally normalize) + +- **Idea (two sizes).** *Small & backward-compatible:* add a `jsonb_path_ops` **GIN index on + `(partitioning -> 'keysToValuesMap')`** plus a helper `get_partitionings_by_keys(...) WHERE + partitioning -> 'keysToValuesMap' @> i_keys`. *Larger:* maintain a normalized child table + `runs.partitioning_keys(fk_partitioning, key, value)` (a trigger keeps it in sync), or generated columns for the + hot linkage keys (`catalog_path`, `info_date`). +- **Unlocks.** Subset / containment lookup ("all currencies"), query-by-dimension ("every partitioning where + `currency = EUR`"), and partial-parent lookup (§5) become **indexed** SQL instead of scans. This is the single + change that turns ADR 03 **Option C** and ADR 04 **§5's hard case** from "new subsystem" into "one index + one + function." +- **Cost.** The GIN index is nearly free and fully backward-compatible (no write-path change). The normalized + table costs a trigger, a backfill, and storage, and must be kept consistent with the JSONB. +- **Verdict.** **Do the GIN index first** — highest value-to-effort in the whole list. Promote to a normalized + table only if dimension queries become a first-class, high-volume access pattern. + +### Bet 2 — Make time a first-class checkpoint dimension + +- **Idea.** The infrastructure behind §4: add a **BRIN index on `runs.checkpoints (process_start_time)`** (ideal + for an append-only, time-ordered table and physically tiny), and consider native **range partitioning** of + `checkpoints` by time once volume warrants. Pair with the `from`/`to` params from §4. +- **Unlocks.** Cheap time-window audits **at scale** — the literal §1 question ("last 2 months") answered + server-side instead of by client over-fetch — and old partitions become trivially archivable. +- **Cost.** BRIN index: low and backward-compatible. Table partitioning: medium (migration), defer until needed. +- **Verdict.** **Do the BRIN index with §4.** It is the cheapest structural change that scales the core audit + query. + +### Bet 3 — Persist lineage as explicit edges (a real DAG) + +- **Idea.** Add `runs.partitioning_edges(fk_parent, fk_child, created_by, created_at)` capturing the **immediate** + parent->child relationship at link time. Keep `flows` / `partitioning_to_flow` as a denormalized reachability + cache - or, longer term, *derive* it from edges. +- **Unlocks.** The exact **edge list** and immediate parent/child (impossible today, per the framing note above), + arbitrary **N-hop traversal** via a recursive CTE, a true multi-parent DAG **with provenance** (who linked what, + when), and a real graph endpoint (`nodes + edges`) for audits that must *show the lineage*, not just the set. +- **Cost.** One new table plus dual-maintenance with flows (medium), or a larger refactor if flows become a pure + projection of edges (higher). +- **Verdict.** Recommend at minimum *recording* edges now (cheap, additive) even before building graph queries — the + provenance is impossible to reconstruct later. + +### Summary + +**Recommended sequence.** Ship the two low-cost, backward-compatible indexes first — **Bet 1's GIN** and **Bet 2's +BRIN** (no producer changes, and together they unlock most of the audit + partial-lookup value). Start **recording +edges** (Bet 3) additively so provenance isn't lost. + +--- + +## 7. Alternatives Considered + +- **Client-side windowing only (status quo).** Page latest-first and stop at the cutoff. Zero server change; + acceptable as an interim. Downside: over-fetches whole pages near the boundary, and every consumer must + re-implement the cutoff. +- **Iterate per partitioning** (`get_partitioning_checkpoints` for each dataset). Rejected: the caller would first + have to enumerate the flow's datasets and then fan out N calls, re-implementing what `get_flow_checkpoints` + already does server-side in one call. +- **A new bespoke "audit export" endpoint.** Rejected for now: the flow-checkpoints endpoint plus a time filter + already returns exactly the needed shape; a dedicated export can come later if a specific report format is + required. + +--- + +## 8. Consequences & Open Questions + +- **Which timestamp is authoritative for "the last 2 months"?** `process_start_time` (when the data was *processed*) vs. + `created_at` (when the checkpoint was *reported to Atum*). Recommendation: filter on `process_start_time` + (business time), but confirm with the audit owners as for late/backfilled reports the two can differ. +- **Volume at scale.** A long-lived flow accumulates many checkpoints; the proposed index is what keeps a + time-window audit query cheap. +- **Explicit parent -> child edges.** Flow membership gives the connected lineage *set*; if an audit needs the exact + edge list (who is parent of whom), that is reconstructed today via `get_partitioning_ancestors` per node. A + dedicated "flow graph" (nodes + edges) endpoint is a possible small future addition. +- **Access control** for audit consumers (read-only scopes/authorization) is out of scope here. From 8e950cf0e96d5cc29cb4852b8d2ff11931541e79 Mon Sep 17 00:00:00 2001 From: lsulak Date: Fri, 25 Sep 2026 23:16:54 +0200 Subject: [PATCH 06/21] feat: querying improvements --- .planning/PROJECT.md | 40 +++-- README.md | 33 ++++ adrs/02-consumption-pattern-for-audits.md | 59 ++++--- .../flows/V0.8.0.1__get_flow_checkpoints.sql | 80 +++++++++- ...V0.8.0.2__get_partitioning_checkpoints.sql | 77 +++++++++- ...ckpoints_partitioning_start_time_index.ddl | 23 +++ ...checkpoint_properties_checkpoint_index.ddl | 23 +++ .../GetFlowCheckpointsIntegrationTests.scala | 144 +++++++++++++++++- ...titioningCheckpointsIntegrationTests.scala | 116 +++++++++++++- .../za/co/absa/atum/reader/FlowReader.scala | 90 ++++++----- .../absa/atum/reader/PartitioningReader.scala | 79 ++++++---- .../za/co/absa/atum/reader/core/Reader.scala | 22 +++ .../reader/requests/CheckpointFilter.scala | 78 ++++++++++ .../reader/requests/QueryParamNames.scala | 3 + .../atum/reader/FlowReaderUnitTests.scala | 97 +++++++++++- .../reader/PartitioningReaderUnitTests.scala | 56 ++++++- .../requests/CheckpointFilterUnitTests.scala | 65 ++++++++ .../flows/functions/GetFlowCheckpoints.scala | 12 +- .../GetPartitioningCheckpoints.scala | 10 +- .../v2/controller/CheckpointController.scala | 3 + .../controller/CheckpointControllerImpl.scala | 5 + .../api/v2/controller/FlowController.scala | 5 + .../v2/controller/FlowControllerImpl.scala | 17 ++- .../atum/server/api/v2/http/Endpoints.scala | 37 +++-- .../v2/repository/CheckpointRepository.scala | 3 + .../repository/CheckpointRepositoryImpl.scala | 14 +- .../api/v2/repository/FlowRepository.scala | 5 + .../v2/repository/FlowRepositoryImpl.scala | 9 +- .../api/v2/service/CheckpointService.scala | 3 + .../v2/service/CheckpointServiceImpl.scala | 5 + .../server/api/v2/service/FlowService.scala | 5 + .../api/v2/service/FlowServiceImpl.scala | 17 ++- ...CheckpointItemWithPartitioningFromDB.scala | 6 +- .../GetFlowCheckpointsIntegrationTests.scala | 29 +++- ...titioningCheckpointsIntegrationTests.scala | 14 +- .../CheckpointControllerUnitTests.scala | 6 +- .../controller/FlowControllerUnitTests.scala | 12 +- .../GetFlowCheckpointsEndpointUnitTests.scala | 88 ++++++++++- ...itioningCheckpointsEndpointUnitTests.scala | 72 ++++++++- .../CheckpointRepositoryUnitTests.scala | 14 +- .../repository/FlowRepositoryUnitTests.scala | 14 +- .../service/CheckpointServiceUnitTests.scala | 8 +- .../api/v2/service/FlowServiceUnitTests.scala | 8 +- ...tItemWithPartitioningFromDBUnitTests.scala | 22 ++- 44 files changed, 1336 insertions(+), 192 deletions(-) create mode 100644 database/src/main/postgres/runs/V0.8.0.3__checkpoints_partitioning_start_time_index.ddl create mode 100644 database/src/main/postgres/runs/V0.8.0.4__checkpoint_properties_checkpoint_index.ddl create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala create mode 100644 reader/src/test/scala/za/co/absa/atum/reader/requests/CheckpointFilterUnitTests.scala diff --git a/.planning/PROJECT.md b/.planning/PROJECT.md index 1f6985322..9790da28e 100644 --- a/.planning/PROJECT.md +++ b/.planning/PROJECT.md @@ -1,36 +1,54 @@ # Project Context **Core Value:** -The Atum system must support efficient, paginated querying of checkpoints by combining both checkpoint name and multi-value properties (e.g., `executionID IN (...)`), avoiding sequential single-property queries. +Atum must answer audit questions efficiently: in one filtered, paginated query, return the last *N* months of +measurements for a dataset *and its lineage*, with each measurement tagged with its dataset. Checkpoint filters +(name, multi-value properties such as `executionID IN (...)`, time window) combine server-side instead of being +applied one after another by the client. **What This Is:** -An enhancement to both `atum-reader` and `atum-server` to expand checkpoint filtering capabilities. +Enhancements to `database`, `atum-server` and `atum-reader` that implement the consumption pattern in +`adrs/02-consumption-pattern-for-audits.md`: server-side time windows, a unified reader filter API, lineage-complete +audit scope, partial-key parent lookup, and recording explicit lineage edges. Milestone v1.1 delivers the +first two; the rest is deferred. + +**Current milestone:** v1.1 Audit Consumption (ADR 002). See `.planning/ROADMAP.md` and `.planning/REQUIREMENTS.md`. **Constraints:** -- Must maintain backward compatibility or provide a clear migration path for existing API consumers. -- Server-side changes required (updating `checkpointProperties` from `Map[String, String]` to support multi-value matching, e.g., `Map[String, Seq[String]]` or a dedicated field). +- Must stay backward compatible for API and reader consumers. New query params and DB function params are optional + and trailing; deprecate before removing. +- A DB migration must not break a server that is still on the previous release (new function params get defaults). +- No re-architecture of checkpoint storage in this milestone. Table partitioning and normalized keys are deferred. ## Requirements ### Validated - ✓ Read checkpoints by exact name (existing) -- ✓ Read checkpoints by single-value properties map (existing) - ✓ Server-side pagination (existing) +- ✓ Server filters checkpoints by multi-value properties, e.g. `executionID IN (a,b,c)` — v1.0, CHK-01/02 +- ✓ Pagination is accurate over the combined name + multi-property filter — v1.0, CHK-03 ### Active -- [ ] Reader exposes a single `getCheckpointsPage` method that accepts both `checkpointName` and `checkpointProperties`. -- [ ] Server API allows filtering checkpoints by a list/set of values for a property (e.g., `executionID IN (a,b,c)`). -- [ ] Reader exposes the multi-value property filtering capability in its API. -- [ ] Server API applies pagination (`limit`, `offset`) across the combined (name + multi-property) result set accurately. +- [ ] Server-side `from`/`to` time window on flow and partitioning checkpoint queries, backed by an index (Phase 1) +- [ ] Reader: one `getCheckpointsPage` with a combined name + multi-value properties + time-window filter, plus a + drain-all-pages helper (Phase 2; carries over v1.0 CHK-04/05) ### Out of Scope - Major architectural changes to checkpoint storage. +- Lineage-complete audit scope, partial-key partitioning search, recording lineage edges (deferred 2026-09-25). +- Access control for audit consumers. +- Changes to the agent's checkpoint write path. ## Key Decisions | Decision | Rationale | Outcome | |----------|-----------|---------| -| Update both server and reader | Server currently only supports `Map[String, String]`, so we must add backend support for IN-lists before the reader can expose it. | — Pending | +| Update both server and reader | Server only supported `Map[String, String]`, so backend IN-list support had to come before the reader could expose it. | ✓ Server done (v1.0); reader in v1.1 Phase 2 | +| Composite B-tree `(fk_partitioning, process_start_time)` before BRIN | Both endpoints scan checkpoints per partitioning. `process_start_time` is client-supplied, so the physical ordering BRIN relies on is not guaranteed. | ✓ Adopted; BRIN deferred | +| Main flow ≠ lineage chain | Verified in SQL: `main-flow(X) = {X} ∪ descendants(X)`. ADR §3 step 2 misses the upstream unless X is a root. | Documented as a known limitation; fix deferred | +| Time window on `process_start_time` | Business time is what audits ask about. | ✓ Decided 2026-09-25 | +| Amend unreleased `V0.8.0.1`/`V0.8.0.2` in place | One function version per release. | ✓ Decided 2026-09-25 | +| `CREATE INDEX CONCURRENTLY`, one index per migration | No write lock on `checkpoints` while the index builds. | ✓ Decided 2026-09-25 | ## Evolution @@ -50,4 +68,4 @@ This document evolves at phase transitions and milestone boundaries. 4. Update Context with current state --- -*Last updated: 2026-09-24 after initialization* +*Last updated: 2026-09-25 — milestone v1.1 (ADR 002) started* diff --git a/README.md b/README.md index 966e559f9..c23482cea 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ - [Usage](#usage) - [Atum Agent routines](#atum-agent-routines) - [Control measurement types](#control-measurement-types) + - [Reading checkpoints](#reading-checkpoints) - [How to generate Code coverage report](#how-to-generate-code-coverage-report) - [How to Run in IntelliJ](#how-to-run-in-intellij) - [How to Run Tests](#how-to-run-tests) @@ -231,6 +232,38 @@ represents all currently supported measurement types (aka measures): [//]: # (| controlType.absAggregatedTruncTotal | Calculates SUM(TRUNC(ABS())) of the specified column |) +### Reading checkpoints + +The _Atum Reader_ reads checkpoints of a partitioning (`PartitioningReader`) or of the whole flow a partitioning is the +main partitioning of (`FlowReader`). The checkpoints to read are described by a `CheckpointFilter`; its parts are all +optional and combined together: + +| Filter part | Meaning | +|---------------|:-----------------------------------------------------------------------------------------------| +| `name` | checkpoints of this name | +| `properties` | for every property name given, the checkpoint property has one of the values (`IN (...)`) | +| `from`, `to` | process start time window: `from` inclusive, `to` exclusive | +| `latestFirst` | order by process start time, latest first (default) or earliest first | + +For example, all measurements of a flow from the last two months, of two specific executions: + +```scala +import za.co.absa.atum.reader.FlowReader +import za.co.absa.atum.reader.requests.CheckpointFilter + +val filter = CheckpointFilter( + properties = Map("executionID" -> Set("a", "b")), + from = Some(ZonedDateTime.now().minusMonths(2)) +) +val reader = FlowReader(mainFlowPartitioning) + +reader.getCheckpointsPage(pageSize = 100, offset = 0, filter = filter) // one page +reader.getAllCheckpoints(filter) // all pages, held in memory +``` + +Each flow checkpoint carries the partitioning it belongs to. The flow of a partitioning covers the partitioning and the +partitionings derived from it (its descendants), not its ancestors. + ## How to generate Code coverage report diff --git a/adrs/02-consumption-pattern-for-audits.md b/adrs/02-consumption-pattern-for-audits.md index e640f408a..5afdcf88c 100644 --- a/adrs/02-consumption-pattern-for-audits.md +++ b/adrs/02-consumption-pattern-for-audits.md @@ -1,4 +1,4 @@ -# ADR 03 — Consumption Pattern for Audit Queries (Time-Windowed Lineage & Measurements) +# ADR 02 — Consumption Pattern for Audit Queries (Time-Windowed Lineage & Measurements) | | | |-------------------------|------------------------------------------------------------------------------------------------------------------| @@ -6,6 +6,7 @@ | **Date** | 2026-09-20 | | **Deciders** | Atum Service maintainers, platform architects, audit/consumer stakeholders | | **Affected components** | `server/` + `database/` (optional time-filter enhancement only), `reader/` (consumption client — already exists) | +| **Implementation** | §4 (time window, index, `latest-first` on flows) and its reader support done (2026-09-25); §5 and §6 deferred | --- @@ -31,9 +32,11 @@ addition that makes the time window efficient. Three facts about the current schema make this almost entirely a "use what's there" exercise: -- **A "flow" is one lineage chain.** When partitionings are linked parent→child (including across applications), - they share **flow membership** (`flows.partitioning_to_flow`). So *one flow = one connected - lineage graph* — exactly the "set of related datasets" an audit cares about. +- **A flow is a partitioning plus everything derived from it.** Every partitioning gets its own *main flow* when it + is created (`flows._create_flow`), and linking a child to a parent (including across applications) adds the child + to **all flows of the parent** (`flows._add_to_parent_flows` → `flows.partitioning_to_flow`). So the main flow of a + partitioning = the partitioning **and its descendants** — the "set of related datasets" downstream of it. It does + **not** contain the partitioning's ancestors; only the main flow of a root covers the whole lineage chain. - **One call returns every measurement in that chain, already tagged with its dataset.** `flows.get_flow_checkpoints(flowId, ...)` returns all checkpoints across the whole flow, and **every returned row carries the `id_partitioning` + `partitioning` JSON it belongs to** — plus `measure_name`, `measurement_value`, @@ -47,7 +50,7 @@ A ready-made client also already exists: the **`reader`** module's `FlowReader`, partitioning → flow → paged-checkpoints walk described below. ``` - Flow = one lineage chain (one connected component) + Main flow of "AQ dataset" = AQ dataset + everything derived from it AQ dataset ─────▶ UU domain ─────▶ UU feed │ │ │ @@ -56,6 +59,8 @@ partitioning → flow → paged-checkpoints walk described below. GET /flows/{flowId}/checkpoints ── returns ALL of these rows, each tagged with the partitioning it belongs to. + + The main flow of "UU feed", however, contains only UU feed (and whatever is derived from it). ``` --- @@ -69,6 +74,10 @@ Three steps, all on **today's** v2 API: point from a business key such as `catalog_path`)* 2. **Resolve its lineage chain → flow id.** `GET /api/v2/partitionings/{id}/main-flow` → `{ id: }`. + This covers the dataset and its **descendants**. To include the **upstream** datasets too, start from the root: + `GET /api/v2/partitionings/{id}/ancestors` lists the primary partitionings of the flows the dataset belongs to; + take the root among them and use *its* main flow. (Known limitation: a partitioning re-parented with + `PATCH .../ancestors` joins the new parent's flows, but its already existing descendants do not.) 3. **Pull all measurements across the chain, newest first, paged.** `GET /api/v2/flows/{flowId}/checkpoints?limit=100&offset=0&include-properties=true`. Each row = one measure of one checkpoint, **tagged with its partitioning**. That is the audit dataset: @@ -114,7 +123,8 @@ If an audit wants the dataset graph *without* the measurements, use `GET /api/v2 **Gap:** there is **no server-side time filter** on any checkpoint endpoint today. `get_flow_checkpoints` filters by `checkpoint-name` and `checkpoint-properties` only; "last 2 months" must be done client-side (above). -**Proposed change — minimal and backward-compatible** (mirrors the *existing* optional filters, so nothing breaks): +**Proposed change — minimal and backward-compatible** (mirrors the *existing* optional filters, so nothing breaks). +*Implemented (2026-09-25), with the index choice described below:* - **Database:** add two optional parameters to `flows.get_flow_checkpoints` (and, for the single-dataset case, `runs.get_partitioning_checkpoints`): @@ -135,16 +145,22 @@ by `checkpoint-name` and `checkpoint-properties` only; "last 2 months" must be d GET /api/v2/flows/{flowId}/checkpoints?from=2026-06-01T00:00:00Z&to=2026-08-01T00:00:00Z&limit=100 ``` -- **Index:** add a supporting index on `runs.checkpoints (process_start_time)` — today the only index on that - table is on `fk_partitioning`, so a large flow's time-window scan has nothing to lean on. A composite - `(fk_partitioning, process_start_time)` also helps the single-partitioning endpoint. +- **Index:** add a supporting index — today the only index on `runs.checkpoints` is on `fk_partitioning`, so a large + flow's time-window scan has nothing to lean on. *Implemented as the composite `(fk_partitioning, process_start_time)` + (`checkpoints_idx2`, built `CONCURRENTLY`)*: both endpoints read checkpoints per partitioning, so each partitioning + of the flow is range-scanned by time. On 2M checkpoints (200k in the flow) a 2-month page of 100 took ~26 ms with + it and ~1.2 s without it. The checkpoint properties filter got a supporting + `runs.checkpoint_properties (fk_checkpoint, property_name)` index as well. With this, "last 2 months" is a single server-side query per page instead of client-side over-fetching, and the cutoff logic lives in one place. -*(Minor, optional: the `/flows/{flowId}/checkpoints` endpoint does not currently expose `latest-first` — the DB -function has it and defaults to `TRUE`, which is what audits want anyway. Expose it only if ascending order is ever -needed.)* +*(Minor: the `/flows/{flowId}/checkpoints` endpoint now exposes `latest-first` too — the DB function has it and +defaults to `TRUE`, which is what audits want anyway.)* + +**Consumption via the reader:** `FlowReader` / `PartitioningReader` take a `CheckpointFilter` (name, multi-value +properties, `from` / `to`, `latestFirst`) in `getCheckpointsPage`, and `getAllCheckpoints` reads all the pages of a +filtered query. --- @@ -197,12 +213,12 @@ containment** lookup is for: — containment must target the `keysToValuesMap` sub-object, because the stored value is the envelope `{ keys, version, keysToValuesMap }` (with an ordered `keys` array), **not** a flat map — backed by a `jsonb_path_ops` **GIN index** on `(partitioning -> 'keysToValuesMap')` (none exists today — the store is - exact-match only, ADR 03 §5.4). + exact-match only). - **API:** `GET /api/v2/partitionings/search?keys=&limit=&offset=` (mirrors the existing base64-JSON convention of `GET /partitionings`). - **0 / 1 / many results:** a partial key legitimately matches many parents — link to **all** of them (the flow model is many-to-many, so N parents need no schema change), or, for the least-selective `catalog_path`-only - case, the most-recent — per ADR 03 §3.2's match-resolution strategy. + case, the most recent one. **Recommendation:** prefer the rollup convention; treat containment search as the fallback for heterogeneous or uncooperative producers (and ad-hoc audits). @@ -237,7 +253,7 @@ Two facts (both verified in the schema) frame the whole list: hot linkage keys (`catalog_path`, `info_date`). - **Unlocks.** Subset / containment lookup ("all currencies"), query-by-dimension ("every partitioning where `currency = EUR`"), and partial-parent lookup (§5) become **indexed** SQL instead of scans. This is the single - change that turns ADR 03 **Option C** and ADR 04 **§5's hard case** from "new subsystem" into "one index + one + change that turns **§5's hard case** (containment search) from "new subsystem" into "one index + one function." - **Cost.** The GIN index is nearly free and fully backward-compatible (no write-path change). The normalized table costs a trigger, a backfill, and storage, and must be kept consistent with the JSONB. @@ -254,6 +270,10 @@ Two facts (both verified in the schema) frame the whole list: - **Cost.** BRIN index: low and backward-compatible. Table partitioning: medium (migration), defer until needed. - **Verdict.** **Do the BRIN index with §4.** It is the cheapest structural change that scales the core audit query. +- **Outcome (2026-09-25).** §4 shipped with the composite B-tree `(fk_partitioning, process_start_time)` instead: + checkpoint queries are always scoped to partitionings, and `process_start_time` is supplied by the client (late or + backfilled reports), so the physical time ordering BRIN depends on is not guaranteed. BRIN is deferred until + `EXPLAIN` at production volume shows a need. ### Bet 3 — Persist lineage as explicit edges (a real DAG) @@ -272,7 +292,8 @@ Two facts (both verified in the schema) frame the whole list: **Recommended sequence.** Ship the two low-cost, backward-compatible indexes first — **Bet 1's GIN** and **Bet 2's BRIN** (no producer changes, and together they unlock most of the audit + partial-lookup value). Start **recording -edges** (Bet 3) additively so provenance isn't lost. +edges** (Bet 3) additively so provenance isn't lost. *(Bet 2 was covered by the composite B-tree of §4; Bets 1 and 3 +are deferred.)* --- @@ -293,11 +314,11 @@ edges** (Bet 3) additively so provenance isn't lost. ## 8. Consequences & Open Questions - **Which timestamp is authoritative for "the last 2 months"?** `process_start_time` (when the data was *processed*) vs. - `created_at` (when the checkpoint was *reported to Atum*). Recommendation: filter on `process_start_time` - (business time), but confirm with the audit owners as for late/backfilled reports the two can differ. + `created_at` (when the checkpoint was *reported to Atum*). **Decided (2026-09-25): `process_start_time`** (business + time). For late/backfilled reports the two can differ. - **Volume at scale.** A long-lived flow accumulates many checkpoints; the proposed index is what keeps a time-window audit query cheap. -- **Explicit parent -> child edges.** Flow membership gives the connected lineage *set*; if an audit needs the exact +- **Explicit parent -> child edges.** Flow membership gives the downstream lineage *set*; if an audit needs the exact edge list (who is parent of whom), that is reconstructed today via `get_partitioning_ancestors` per node. A dedicated "flow graph" (nodes + edges) endpoint is a possible small future addition. - **Access control** for audit consumers (read-only scopes/authorization) is out of scope here. diff --git a/database/src/main/postgres/flows/V0.8.0.1__get_flow_checkpoints.sql b/database/src/main/postgres/flows/V0.8.0.1__get_flow_checkpoints.sql index f3192a218..74c90e200 100644 --- a/database/src/main/postgres/flows/V0.8.0.1__get_flow_checkpoints.sql +++ b/database/src/main/postgres/flows/V0.8.0.1__get_flow_checkpoints.sql @@ -1,3 +1,21 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- The previous overload is dropped so that the new signature (JSONB multi-value checkpoint properties filter and +-- the process start time window) is unambiguous when the function is called with named or defaulted arguments. DROP FUNCTION IF EXISTS flows.get_flow_checkpoints(BIGINT, INT, BIGINT, TEXT, HSTORE, BOOLEAN); CREATE OR REPLACE FUNCTION flows.get_flow_checkpoints( @@ -7,6 +25,8 @@ CREATE OR REPLACE FUNCTION flows.get_flow_checkpoints( IN i_checkpoint_name TEXT DEFAULT NULL, IN i_checkpoint_properties JSONB DEFAULT NULL, IN i_latest_first BOOLEAN DEFAULT TRUE, + IN i_from_time TIMESTAMP WITH TIME ZONE DEFAULT NULL, + IN i_to_time TIMESTAMP WITH TIME ZONE DEFAULT NULL, OUT status INTEGER, OUT status_text TEXT, OUT id_checkpoint UUID, @@ -22,8 +42,56 @@ CREATE OR REPLACE FUNCTION flows.get_flow_checkpoints( OUT partitioning JSONB, OUT partitioning_author TEXT, OUT has_more BOOLEAN -) - RETURNS SETOF record AS +) RETURNS SETOF record AS +-------------------------------------------------------------------------------------------------------------------- +-- +-- Function: flows.get_flow_checkpoints(8) +-- Retrieves all checkpoints (measures and their measurement details) related to an input flow (and checkpoint +-- name, checkpoint properties and/or process start time window, if specified). +-- +-- Note: a single row returned from this function doesn't contain all data related to a single checkpoint - it only +-- represents one measure associated with a checkpoint. So even if only a single checkpoint would be retrieved, +-- this function can potentially return multiple rows. +-- +-- Parameters: +-- i_flow_id - ID of the flow for which checkpoints are to be retrieved +-- i_checkpoints_limit - (optional) maximum number of checkpoints to return, returns all of them if NULL +-- i_offset - (optional) offset of the first checkpoint to return +-- i_checkpoint_name - (optional) if specified, returns data related to particular checkpoint's name +-- i_checkpoint_properties - (optional) JSON object mapping a property name to an array of accepted values, +-- e.g. {"executionID": ["a", "b"]}; returns only checkpoints that, for every given +-- property name, have that property with one of the accepted values +-- i_latest_first - (optional) if true (default), checkpoints are ordered by process_start_time +-- in descending order (latest first); if false, in ascending order +-- i_from_time - (optional) if specified, returns only checkpoints with process_start_time >= i_from_time +-- i_to_time - (optional) if specified, returns only checkpoints with process_start_time < i_to_time +-- +-- Note: i_checkpoints_limit and i_offset are used for pagination purposes; +-- checkpoints are ordered by process_start_time (descending by default, see i_latest_first) +-- and then by id_checkpoint in ascending order as a tie-breaker +-- +-- Returns: +-- status - Status code +-- status_text - Status text +-- id_checkpoint - ID of retrieved checkpoint +-- checkpoint_name - Name of the retrieved checkpoint +-- checkpoint_author - Author of the checkpoint +-- measured_by_atum_agent - Flag indicating whether the checkpoint was measured by Atum Agent +-- (if false, data supplied manually) +-- measure_name - measure name associated with a given checkpoint +-- measured_columns - measure columns associated with a given checkpoint +-- measurement_value - measurement details associated with a given checkpoint +-- checkpoint_start_time - Time of the checkpoint +-- checkpoint_end_time - End time of the checkpoint computation +-- id_partitioning - ID of the partitioning +-- partitioning - Partitioning value +-- partitioning_author - Author of the partitioning +-- has_more - Flag indicating whether there are more checkpoints available, always `false` if `i_limit` is NULL +-- +-- Status codes: +-- 11 - OK +-- 42 - Flow not found +--------------------------------------------------------------------------------------------------- $$ DECLARE _has_more BOOLEAN; @@ -51,6 +119,8 @@ BEGIN JOIN flows.partitioning_to_flow PF ON C.fk_partitioning = PF.fk_partitioning WHERE PF.fk_flow = i_flow_id AND (i_checkpoint_name IS NULL OR C.checkpoint_name = i_checkpoint_name) + AND (i_from_time IS NULL OR C.process_start_time >= i_from_time) + AND (i_to_time IS NULL OR C.process_start_time < i_to_time) AND ( i_checkpoint_properties IS NULL OR NOT EXISTS ( @@ -91,6 +161,8 @@ BEGIN JOIN flows.partitioning_to_flow PF ON C.fk_partitioning = PF.fk_partitioning WHERE PF.fk_flow = i_flow_id AND (i_checkpoint_name IS NULL OR C.checkpoint_name = i_checkpoint_name) + AND (i_from_time IS NULL OR C.process_start_time >= i_from_time) + AND (i_to_time IS NULL OR C.process_start_time < i_to_time) AND ( i_checkpoint_properties IS NULL OR NOT EXISTS ( @@ -145,5 +217,5 @@ BEGIN END; $$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER; -ALTER FUNCTION flows.get_flow_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN) OWNER TO atum_owner; -GRANT EXECUTE ON FUNCTION flows.get_flow_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN) TO atum_owner; +ALTER FUNCTION flows.get_flow_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE) OWNER TO atum_owner; +GRANT EXECUTE ON FUNCTION flows.get_flow_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE) TO atum_owner; diff --git a/database/src/main/postgres/runs/V0.8.0.2__get_partitioning_checkpoints.sql b/database/src/main/postgres/runs/V0.8.0.2__get_partitioning_checkpoints.sql index 6f4860a47..76ad85905 100644 --- a/database/src/main/postgres/runs/V0.8.0.2__get_partitioning_checkpoints.sql +++ b/database/src/main/postgres/runs/V0.8.0.2__get_partitioning_checkpoints.sql @@ -1,3 +1,21 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- The previous overload is dropped so that the new signature (JSONB multi-value checkpoint properties filter and +-- the process start time window) is unambiguous when the function is called with named or defaulted arguments. DROP FUNCTION IF EXISTS runs.get_partitioning_checkpoints(BIGINT, INT, BIGINT, TEXT, HSTORE, BOOLEAN); CREATE OR REPLACE FUNCTION runs.get_partitioning_checkpoints( @@ -7,6 +25,8 @@ CREATE OR REPLACE FUNCTION runs.get_partitioning_checkpoints( IN i_checkpoint_name TEXT DEFAULT NULL, IN i_checkpoint_properties JSONB DEFAULT NULL, IN i_latest_first BOOLEAN DEFAULT TRUE, + IN i_from_time TIMESTAMP WITH TIME ZONE DEFAULT NULL, + IN i_to_time TIMESTAMP WITH TIME ZONE DEFAULT NULL, OUT status INTEGER, OUT status_text TEXT, OUT id_checkpoint UUID, @@ -19,8 +39,53 @@ CREATE OR REPLACE FUNCTION runs.get_partitioning_checkpoints( OUT checkpoint_start_time TIMESTAMP WITH TIME ZONE, OUT checkpoint_end_time TIMESTAMP WITH TIME ZONE, OUT has_more BOOLEAN -) - RETURNS SETOF record AS +) RETURNS SETOF record AS +-------------------------------------------------------------------------------------------------------------------- +-- +-- Function: runs.get_partitioning_checkpoints(8) +-- Retrieves all checkpoints (measures and their measurement details) related to an input partitioning (and +-- checkpoint name, checkpoint properties and/or process start time window, if specified). +-- +-- Note: a single row returned from this function doesn't contain all data related to a single checkpoint - it only +-- represents one measure associated with a checkpoint. So even if only a single checkpoint would be retrieved, +-- this function can potentially return multiple rows. +-- +-- Parameters: +-- i_partitioning_id - ID of the partitioning for which checkpoints are to be retrieved +-- i_checkpoints_limit - (optional) maximum number of checkpoints to return, returns all of them if NULL +-- i_offset - (optional) offset of the first checkpoint to return +-- i_checkpoint_name - (optional) if specified, returns data related to particular checkpoint's name +-- i_checkpoint_properties - (optional) JSON object mapping a property name to an array of accepted values, +-- e.g. {"executionID": ["a", "b"]}; returns only checkpoints that, for every given +-- property name, have that property with one of the accepted values +-- i_latest_first - (optional) if true (default), checkpoints are ordered by process_start_time +-- in descending order (latest first); if false, in ascending order +-- i_from_time - (optional) if specified, returns only checkpoints with process_start_time >= i_from_time +-- i_to_time - (optional) if specified, returns only checkpoints with process_start_time < i_to_time +-- +-- Note: i_checkpoints_limit and i_offset are used for pagination purposes; +-- checkpoints are ordered by process_start_time (descending by default, see i_latest_first) +-- and then by id_checkpoint in ascending order as a tie-breaker +-- +-- Returns: +-- status - Status code +-- status_text - Status text +-- id_checkpoint - ID of retrieved checkpoint +-- checkpoint_name - Name of the retrieved checkpoint +-- checkpoint_author - Author of the checkpoint +-- measured_by_atum_agent - Flag indicating whether the checkpoint was measured by Atum Agent +-- (if false, data supplied manually) +-- measure_name - measure name associated with a given checkpoint +-- measured_columns - measure columns associated with a given checkpoint +-- measurement_value - measurement details associated with a given checkpoint +-- checkpoint_start_time - Time of the checkpoint +-- checkpoint_end_time - End time of the checkpoint computation +-- has_more - Flag indicating whether there are more checkpoints available +-- +-- Status codes: +-- 11 - OK +-- 41 - Partitioning not found +--------------------------------------------------------------------------------------------------- $$ DECLARE _has_more BOOLEAN; @@ -41,6 +106,8 @@ BEGIN FROM runs.checkpoints C WHERE C.fk_partitioning = i_partitioning_id AND (i_checkpoint_name IS NULL OR C.checkpoint_name = i_checkpoint_name) + AND (i_from_time IS NULL OR C.process_start_time >= i_from_time) + AND (i_to_time IS NULL OR C.process_start_time < i_to_time) AND ( i_checkpoint_properties IS NULL OR NOT EXISTS ( @@ -79,6 +146,8 @@ BEGIN FROM runs.checkpoints C WHERE C.fk_partitioning = i_partitioning_id AND (i_checkpoint_name IS NULL OR C.checkpoint_name = i_checkpoint_name) + AND (i_from_time IS NULL OR C.process_start_time >= i_from_time) + AND (i_to_time IS NULL OR C.process_start_time < i_to_time) AND ( i_checkpoint_properties IS NULL OR NOT EXISTS ( @@ -128,5 +197,5 @@ BEGIN END; $$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER; -ALTER FUNCTION runs.get_partitioning_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN) OWNER TO atum_owner; -GRANT EXECUTE ON FUNCTION runs.get_partitioning_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN) TO atum_owner; +ALTER FUNCTION runs.get_partitioning_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE) OWNER TO atum_owner; +GRANT EXECUTE ON FUNCTION runs.get_partitioning_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE) TO atum_owner; diff --git a/database/src/main/postgres/runs/V0.8.0.3__checkpoints_partitioning_start_time_index.ddl b/database/src/main/postgres/runs/V0.8.0.3__checkpoints_partitioning_start_time_index.ddl new file mode 100644 index 000000000..9e7d84004 --- /dev/null +++ b/database/src/main/postgres/runs/V0.8.0.3__checkpoints_partitioning_start_time_index.ddl @@ -0,0 +1,23 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Supports the process start time window of runs.get_partitioning_checkpoints and flows.get_flow_checkpoints: +-- checkpoints of each partitioning are range-scanned by process_start_time. +-- +-- Built CONCURRENTLY so writes to runs.checkpoints are not blocked. This file must contain only this statement: +-- Flyway then runs it outside of a transaction, which CONCURRENTLY requires. +-- If the build fails, it leaves an INVALID index behind; drop it (DROP INDEX CONCURRENTLY runs.checkpoints_idx2) +-- and re-run the migration. +CREATE INDEX CONCURRENTLY checkpoints_idx2 ON runs.checkpoints (fk_partitioning, process_start_time); diff --git a/database/src/main/postgres/runs/V0.8.0.4__checkpoint_properties_checkpoint_index.ddl b/database/src/main/postgres/runs/V0.8.0.4__checkpoint_properties_checkpoint_index.ddl new file mode 100644 index 000000000..bf7f54a0f --- /dev/null +++ b/database/src/main/postgres/runs/V0.8.0.4__checkpoint_properties_checkpoint_index.ddl @@ -0,0 +1,23 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Supports the checkpoint properties filter of runs.get_partitioning_checkpoints and flows.get_flow_checkpoints +-- (looked up per checkpoint and property name), and the retrieval of a checkpoint's properties. +-- +-- Built CONCURRENTLY so writes to runs.checkpoint_properties are not blocked. This file must contain only this +-- statement: Flyway then runs it outside of a transaction, which CONCURRENTLY requires. +-- If the build fails, it leaves an INVALID index behind; drop it +-- (DROP INDEX CONCURRENTLY runs.idx_checkpoint_properties_checkpoint_name) and re-run the migration. +CREATE INDEX CONCURRENTLY idx_checkpoint_properties_checkpoint_name ON runs.checkpoint_properties (fk_checkpoint, property_name); diff --git a/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala index 62ad5e6f4..f3c71a747 100644 --- a/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala @@ -3,7 +3,7 @@ package za.co.absa.atum.database.flows import io.circe.Json import io.circe.parser.parse import za.co.absa.balta.DBTestSuite -import za.co.absa.balta.classes.JsonBString +import za.co.absa.balta.classes.{JsonBString, QueryResult} import za.co.absa.balta.classes.setter.CustomDBType import java.time.OffsetDateTime @@ -847,6 +847,148 @@ class GetFlowCheckpointsIntegrationTests extends DBTestSuite { } } + test("getFlowCheckpointsV2 should return only checkpoints within the process start time window, across partitionings") { + + val otherPartitioning = JsonBString( + """ + |{ + | "version": 1, + | "keys": ["keyX", "keyY", "keyZ"], + | "keysToValuesMap": { + | "keyX": "value1", + | "keyZ": "value3", + | "keyY": "otherValue2" + | } + |} + |""".stripMargin + ) + + val partitioningIdA: Long = Random.nextLong() + val partitioningIdB: Long = Random.nextLong() + table("runs.partitionings").insert( + add("id_partitioning", partitioningIdA) + .add("partitioning", partitioning) + .add("created_by", "Joseph") + ) + table("runs.partitionings").insert( + add("id_partitioning", partitioningIdB) + .add("partitioning", otherPartitioning) + .add("created_by", "Joseph") + ) + + val flowId: Long = Random.nextLong() + table("flows.flows").insert( + add("id_flow", flowId) + .add("flow_name", "flowNameTimeWindow") + .add("from_pattern", false) + .add("created_by", "Joseph") + .add("fk_primary_partitioning", partitioningIdA) + ) + Seq(partitioningIdA, partitioningIdB).foreach { partitioningId => + table("flows.partitioning_to_flow").insert( + add("fk_flow", flowId) + .add("fk_partitioning", partitioningId) + .add("created_by", "ObviouslySomeTest") + ) + } + + val measureDefinitionIdA: Long = Random.nextLong() + val measureDefinitionIdB: Long = Random.nextLong() + Seq(partitioningIdA -> measureDefinitionIdA, partitioningIdB -> measureDefinitionIdB).foreach { + case (partitioningId, measureDefinitionId) => + table("runs.measure_definitions").insert( + add("id_measure_definition", measureDefinitionId) + .add("fk_partitioning", partitioningId) + .add("measure_name", "cnt") + .add("measured_columns", CustomDBType("""{"col1"}""", "TEXT[]")) + .add("created_by", "Joseph") + ) + } + + // one checkpoint at the start of each month, alternating between the two partitionings of the flow + case class TestCheckpoint(id: UUID, partitioningId: Long, measureDefinitionId: Long, start: OffsetDateTime, executionId: String) + val jan = TestCheckpoint(UUID.randomUUID(), partitioningIdA, measureDefinitionIdA, OffsetDateTime.parse("2026-01-01T00:00:00Z"), "e1") + val feb = TestCheckpoint(UUID.randomUUID(), partitioningIdB, measureDefinitionIdB, OffsetDateTime.parse("2026-02-01T00:00:00Z"), "e1") + val mar = TestCheckpoint(UUID.randomUUID(), partitioningIdA, measureDefinitionIdA, OffsetDateTime.parse("2026-03-01T00:00:00Z"), "e2") + val apr = TestCheckpoint(UUID.randomUUID(), partitioningIdB, measureDefinitionIdB, OffsetDateTime.parse("2026-04-01T00:00:00Z"), "e3") + Seq(jan, feb, mar, apr).foreach { checkpoint => + table("runs.checkpoints").insert( + add("id_checkpoint", checkpoint.id) + .add("fk_partitioning", checkpoint.partitioningId) + .add("checkpoint_name", "CheckpointName") + .add("measured_by_atum_agent", true) + .add("process_start_time", checkpoint.start) + .add("created_by", "Joseph") + ) + table("runs.measurements").insert( + add("fk_measure_definition", checkpoint.measureDefinitionId) + .add("fk_checkpoint", checkpoint.id) + .add("measurement_value", measurementCnt) + ) + table("runs.checkpoint_properties").insert( + add("fk_checkpoint", checkpoint.id) + .add("property_name", "executionID") + .add("property_value", checkpoint.executionId) + ) + } + + def checkpointsOf(queryResult: QueryResult): Seq[(UUID, Long)] = + queryResult.map(row => (row.getUUID("id_checkpoint").get, row.getLong("id_partitioning").get)).toList + + // [feb, apr) spans both partitionings of the flow, latest first + function(fncGetFlowCheckpointsV2) + .setParam("i_flow_id", flowId) + .setParam("i_from_time", feb.start) + .setParam("i_to_time", apr.start) + .execute { queryResult => + assert(checkpointsOf(queryResult) == Seq((mar.id, partitioningIdA), (feb.id, partitioningIdB))) + } + + // the same window, earliest first + function(fncGetFlowCheckpointsV2) + .setParam("i_flow_id", flowId) + .setParam("i_from_time", feb.start) + .setParam("i_to_time", apr.start) + .setParam("i_latest_first", false) + .execute { queryResult => + assert(checkpointsOf(queryResult).map(_._1) == Seq(feb.id, mar.id)) + } + + // combined with a multi-value checkpoint properties filter: executionID IN (e1, e3) since February + function(fncGetFlowCheckpointsV2) + .setParam("i_flow_id", flowId) + .setParam("i_checkpoint_properties", JsonBString("""{"executionID": ["e1", "e3"]}""")) + .setParam("i_from_time", feb.start) + .execute { queryResult => + assert(checkpointsOf(queryResult).map(_._1) == Seq(apr.id, feb.id)) + } + + // paging within the window: has_more is computed over the window only + function(fncGetFlowCheckpointsV2) + .setParam("i_flow_id", flowId) + .setParam("i_checkpoints_limit", 1) + .setParam("i_from_time", feb.start) + .setParam("i_to_time", apr.start) + .execute { queryResult => + val row = queryResult.next() + assert(row.getUUID("id_checkpoint").contains(mar.id)) + assert(row.getBoolean("has_more").contains(true)) + assert(!queryResult.hasNext) + } + function(fncGetFlowCheckpointsV2) + .setParam("i_flow_id", flowId) + .setParam("i_checkpoints_limit", 1) + .setParam("i_offset", 1L) + .setParam("i_from_time", feb.start) + .setParam("i_to_time", apr.start) + .execute { queryResult => + val row = queryResult.next() + assert(row.getUUID("id_checkpoint").contains(feb.id)) + assert(row.getBoolean("has_more").contains(false)) + assert(!queryResult.hasNext) + } + } + private def parseJsonBStringOrThrow(jsonBString: JsonBString): Json = { parse(jsonBString.value).getOrElse(throw new Exception("Failed to parse JsonBString to Json")) } diff --git a/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala index 797b9d77b..110a961dd 100644 --- a/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.database.runs import za.co.absa.balta.DBTestSuite -import za.co.absa.balta.classes.JsonBString +import za.co.absa.balta.classes.{JsonBString, QueryResult} import za.co.absa.balta.classes.setter.CustomDBType import java.time.OffsetDateTime @@ -492,4 +492,118 @@ class GetPartitioningCheckpointsIntegrationTests extends DBTestSuite { } } + test("Returns only checkpoints within the process start time window") { + table("runs.partitionings").insert( + add("partitioning", partitioning1) + .add("created_by", "Daniel") + ) + val fkPartitioning: Long = table("runs.partitionings") + .fieldValue("partitioning", partitioning1, "id_partitioning").get.get + + table("runs.measure_definitions").insert( + add("id_measure_definition", id_measure_definition1) + .add("fk_partitioning", fkPartitioning) + .add("created_by", "Daniel") + .add("measure_name", "measure_1") + .add("measured_columns", measured_columns1) + ) + + // one checkpoint at the start of each month, January to April; named "odd" / "even" by month number + val times = Seq( + OffsetDateTime.parse("2026-01-01T00:00:00Z"), + OffsetDateTime.parse("2026-02-01T00:00:00Z"), + OffsetDateTime.parse("2026-03-01T00:00:00Z"), + OffsetDateTime.parse("2026-04-01T00:00:00Z") + ) + val checkpointIds = times.map(_ => UUID.randomUUID()) + times.zip(checkpointIds).zipWithIndex.foreach { case ((time, checkpointId), i) => + table("runs.checkpoints").insert( + add("id_checkpoint", checkpointId) + .add("fk_partitioning", fkPartitioning) + .add("checkpoint_name", if (i % 2 == 0) "odd" else "even") + .add("process_start_time", time) + .add("process_end_time", endTime) + .add("measured_by_atum_agent", true) + .add("created_by", "Daniel") + ) + table("runs.measurements").insert( + add("fk_checkpoint", checkpointId) + .add("fk_measure_definition", id_measure_definition1) + .add("measurement_value", measurement1) + ) + } + val Seq(jan, feb, mar, apr) = checkpointIds + + def checkpointsOf(queryResult: QueryResult): Seq[UUID] = + queryResult.map(_.getUUID("id_checkpoint").get).toList + + // i_from_time is inclusive + function(fncGetPartitioningCheckpoints) + .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_from_time", times(1)) + .execute { queryResult => + assert(checkpointsOf(queryResult) == Seq(apr, mar, feb)) + } + + // i_to_time is exclusive + function(fncGetPartitioningCheckpoints) + .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_to_time", times(2)) + .execute { queryResult => + assert(checkpointsOf(queryResult) == Seq(feb, jan)) + } + + // both bounds, earliest first + function(fncGetPartitioningCheckpoints) + .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_from_time", times(1)) + .setParam("i_to_time", times(3)) + .setParam("i_latest_first", false) + .execute { queryResult => + assert(checkpointsOf(queryResult) == Seq(feb, mar)) + } + + // paging within the window: has_more is computed over the window only + function(fncGetPartitioningCheckpoints) + .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_checkpoints_limit", 1) + .setParam("i_from_time", times(1)) + .setParam("i_to_time", times(3)) + .execute { queryResult => + val row = queryResult.next() + assert(row.getUUID("id_checkpoint").contains(mar)) + assert(row.getBoolean("has_more").contains(true)) + assert(!queryResult.hasNext) + } + function(fncGetPartitioningCheckpoints) + .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_checkpoints_limit", 1) + .setParam("i_offset", 1L) + .setParam("i_from_time", times(1)) + .setParam("i_to_time", times(3)) + .execute { queryResult => + val row = queryResult.next() + assert(row.getUUID("id_checkpoint").contains(feb)) + assert(row.getBoolean("has_more").contains(false)) + assert(!queryResult.hasNext) + } + + // combined with the checkpoint name filter + function(fncGetPartitioningCheckpoints) + .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_checkpoint_name", "odd") + .setParam("i_from_time", times(1)) + .execute { queryResult => + assert(checkpointsOf(queryResult) == Seq(mar)) + } + + // a window without checkpoints returns no rows + function(fncGetPartitioningCheckpoints) + .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_from_time", OffsetDateTime.parse("2027-01-01T00:00:00Z")) + .execute { queryResult => + assert(!queryResult.hasNext) + } + } + } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala index 05e630a77..f4f8dd8f9 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala @@ -20,13 +20,12 @@ import sttp.client3.SttpBackend import sttp.monad.MonadError import sttp.monad.syntax._ import za.co.absa.atum.model.dto.{CheckpointWithPartitioningDTO, FlowDTO} -import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ import za.co.absa.atum.model.envelopes.SuccessResponse.{PaginatedResponse, SingleSuccessResponse} import za.co.absa.atum.model.types.basic.AtumPartitions import za.co.absa.atum.reader.core.RequestResult.RequestResult import za.co.absa.atum.model.ApiPaths._ import za.co.absa.atum.reader.core.{PartitioningIdProvider, Reader} -import za.co.absa.atum.reader.requests.QueryParamNames +import za.co.absa.atum.reader.requests.{CheckpointFilter, QueryParamNames} import za.co.absa.atum.reader.server.ServerConfig /** @@ -45,32 +44,59 @@ case class FlowReader[F[_]](mainFlowPartitioning: AtumPartitions)(implicit with PartitioningIdProvider[F] { /** - * Function to retrieve a page of checkpoints belonging to the flow. - * The checkpoints are ordered by their creation order. + * Function to retrieve a page of checkpoints belonging to the flow, optionally filtered. + * The checkpoints are ordered by their process start time, latest first unless the filter says otherwise. * - * @param pageSize - the size of the page (record count) to be returned - * @param offset - offset of the page (starting position) + * @param pageSize - the size of the page (record count) to be returned + * @param offset - offset of the page (starting position) * @param includeProperties - whether to include checkpoint properties in the response - * @return - a page of checkpoints + * @param filter - the checkpoints to return (name, properties, process start time window) and their order + * @return - a page of checkpoints */ def getCheckpointsPage( pageSize: Int = 10, offset: Long = 0, - includeProperties: Boolean = false + includeProperties: Boolean = false, + filter: CheckpointFilter = CheckpointFilter.empty ): F[RequestResult[PaginatedResponse[CheckpointWithPartitioningDTO]]] = { for { - mainPartitioningIdOrError <- partitioningId(mainFlowPartitioning) - flowIdOrError <- mapRequestResultF(mainPartitioningIdOrError, queryFlowId) + flowIdOrError <- flowId + checkpointsOrError <- mapRequestResultF( + flowIdOrError, + queryCheckpoints(_, filter, pageSize, offset, includeProperties) + ) + } yield checkpointsOrError + } + + /** + * Function to retrieve all checkpoints belonging to the flow that satisfy the filter, querying them page by page. + * The checkpoints are ordered by their process start time, latest first unless the filter says otherwise. + * + * All matching checkpoints are held in memory, so bound the query with the filter, typically with a process start + * time window, e.g. `CheckpointFilter(from = Some(ZonedDateTime.now().minusMonths(2)))` for the last two months. + * + * @param filter - the checkpoints to return (name, properties, process start time window) and their order + * @param includeProperties - whether to include checkpoint properties in the response + * @param pageSize - the size of the pages (record count) to query the checkpoints in + * @return - all the checkpoints satisfying the filter, or the first error encountered + */ + def getAllCheckpoints( + filter: CheckpointFilter = CheckpointFilter.empty, + includeProperties: Boolean = false, + pageSize: Int = 100 + ): F[RequestResult[Seq[CheckpointWithPartitioningDTO]]] = { + for { + flowIdOrError <- flowId checkpointsOrError <- mapRequestResultF( flowIdOrError, - queryCheckpoints(_, None, Map.empty, pageSize, offset, includeProperties) + (flowId: Long) => queryAllPages(pageSize, queryCheckpoints(flowId, filter, _, _, includeProperties)) ) } yield checkpointsOrError } /** * Function to retrieve a page of checkpoints of the given name belonging to the flow. - * The checkpoints are ordered by their creation order. + * The checkpoints are ordered by their process start time, latest first. * * @param checkpointName - the name to filter with * @param pageSize - the size of the page (record count) to be returned @@ -78,26 +104,20 @@ case class FlowReader[F[_]](mainFlowPartitioning: AtumPartitions)(implicit * @param includeProperties - whether to include checkpoint properties in the response * @return - a page of checkpoints */ + @deprecated("Use getCheckpointsPage with filter = CheckpointFilter(name = Some(checkpointName))", "0.9.0") def getCheckpointsOfNamePage( checkpointName: String, pageSize: Int = 10, offset: Long = 0, includeProperties: Boolean = false ): F[RequestResult[PaginatedResponse[CheckpointWithPartitioningDTO]]] = { - for { - mainPartitioningIdOrError <- partitioningId(mainFlowPartitioning) - flowIdOrError <- mapRequestResultF(mainPartitioningIdOrError, queryFlowId) - checkpointsOrError <- mapRequestResultF( - flowIdOrError, - queryCheckpoints(_, Some(checkpointName), Map.empty, pageSize, offset, includeProperties) - ) - } yield checkpointsOrError + getCheckpointsPage(pageSize, offset, includeProperties, CheckpointFilter(name = Some(checkpointName))) } /** * Function to retrieve a page of checkpoints belonging to the flow that have all the given * checkpoint properties (matching both property name and value). - * The checkpoints are ordered by their creation order. + * The checkpoints are ordered by their process start time, latest first. * * @param checkpointProperties - the checkpoint properties (key-value pairs) to filter with; * a checkpoint is returned only if it has all of them @@ -106,20 +126,15 @@ case class FlowReader[F[_]](mainFlowPartitioning: AtumPartitions)(implicit * @param includeProperties - whether to include checkpoint properties in the response * @return - a page of checkpoints */ + @deprecated("Use getCheckpointsPage with filter = CheckpointFilter(properties = ...)", "0.9.0") def getCheckpointsByPropertiesPage( checkpointProperties: Map[String, String], pageSize: Int = 10, offset: Long = 0, includeProperties: Boolean = false ): F[RequestResult[PaginatedResponse[CheckpointWithPartitioningDTO]]] = { - for { - mainPartitioningIdOrError <- partitioningId(mainFlowPartitioning) - flowIdOrError <- mapRequestResultF(mainPartitioningIdOrError, queryFlowId) - checkpointsOrError <- mapRequestResultF( - flowIdOrError, - queryCheckpoints(_, None, checkpointProperties, pageSize, offset, includeProperties) - ) - } yield checkpointsOrError + val properties = checkpointProperties.map { case (propertyName, value) => propertyName -> Set(value) } + getCheckpointsPage(pageSize, offset, includeProperties, CheckpointFilter(properties = properties)) } /** @@ -137,25 +152,26 @@ case class FlowReader[F[_]](mainFlowPartitioning: AtumPartitions)(implicit } } + private def flowId: F[RequestResult[Long]] = { + for { + mainPartitioningIdOrError <- partitioningId(mainFlowPartitioning) + flowIdOrError <- mapRequestResultF(mainPartitioningIdOrError, queryFlowId) + } yield flowIdOrError + } + private def queryCheckpoints( flowId: Long, - checkpointName: Option[String], - checkpointProperties: Map[String, String], + filter: CheckpointFilter, limit: Int, offset: Long, includeProperties: Boolean ): F[RequestResult[PaginatedResponse[CheckpointWithPartitioningDTO]]] = { val endpoint = s"/$Api/$V2/${V2Paths.Flows}/$flowId/${V2Paths.Checkpoints}" - val propertiesParam: Option[(String, String)] = - if (checkpointProperties.isEmpty) None - else Some(QueryParamNames.CheckpointProperties -> checkpointProperties.asBase64EncodedJsonString) val params = Map( QueryParamNames.Limit -> limit.toString, QueryParamNames.Offset -> offset.toString, QueryParamNames.IncludeProperties -> includeProperties.toString - ) ++ - checkpointName.map(QueryParamNames.CheckpointName -> _) ++ - propertiesParam + ) ++ filter.toQueryParams getQuery(endpoint, params) } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala index a406f0035..6f3dd89af 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala @@ -20,13 +20,12 @@ import sttp.client3.SttpBackend import sttp.monad.MonadError import sttp.monad.syntax._ import za.co.absa.atum.model.ApiPaths.{Api, V2, V2Paths} -import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ import za.co.absa.atum.model.dto.{AdditionalDataDTO, AdditionalDataItemDTO, AdditionalDataItemV2DTO, CheckpointV2DTO} import za.co.absa.atum.model.envelopes.SuccessResponse.{MultiSuccessResponse, PaginatedResponse, SingleSuccessResponse} import za.co.absa.atum.model.types.basic.AtumPartitions import za.co.absa.atum.reader.core.RequestResult.RequestResult import za.co.absa.atum.reader.core.{PartitioningIdProvider, Reader} -import za.co.absa.atum.reader.requests.QueryParamNames +import za.co.absa.atum.reader.requests.{CheckpointFilter, QueryParamNames} import za.co.absa.atum.reader.server.ServerConfig /** @@ -43,24 +42,53 @@ case class PartitioningReader[F[_]](partitioning: AtumPartitions)(implicit with PartitioningIdProvider[F] { /** - * Function to retrieve a page of checkpoints belonging to the partitioning. - * The checkpoints are ordered by their creation order. + * Function to retrieve a page of checkpoints belonging to the partitioning, optionally filtered. + * The checkpoints are ordered by their process start time, latest first unless the filter says otherwise. * - * @param pageSize - the size of the page (record count) to be returned - * @param offset - offset of the page (starting position) + * @param pageSize - the size of the page (record count) to be returned + * @param offset - offset of the page (starting position) * @param includeProperties - whether to include checkpoint properties in the response - * @return - a page of checkpoints + * @param filter - the checkpoints to return (name, properties, process start time window) and their order + * @return - a page of checkpoints */ def getCheckpointsPage( pageSize: Int = 10, offset: Long = 0, - includeProperties: Boolean = false + includeProperties: Boolean = false, + filter: CheckpointFilter = CheckpointFilter.empty ): F[RequestResult[PaginatedResponse[CheckpointV2DTO]]] = { for { partitioningIdOrError <- partitioningId(partitioning) checkpointsOrError <- mapRequestResultF( partitioningIdOrError, - queryCheckpoints(_, None, Map.empty, pageSize, offset, includeProperties) + queryCheckpoints(_, filter, pageSize, offset, includeProperties) + ) + } yield checkpointsOrError + } + + /** + * Function to retrieve all checkpoints belonging to the partitioning that satisfy the filter, querying them page by + * page. The checkpoints are ordered by their process start time, latest first unless the filter says otherwise. + * + * All matching checkpoints are held in memory, so bound the query with the filter, typically with a process start + * time window, e.g. `CheckpointFilter(from = Some(ZonedDateTime.now().minusMonths(2)))` for the last two months. + * + * @param filter - the checkpoints to return (name, properties, process start time window) and their order + * @param includeProperties - whether to include checkpoint properties in the response + * @param pageSize - the size of the pages (record count) to query the checkpoints in + * @return - all the checkpoints satisfying the filter, or the first error encountered + */ + def getAllCheckpoints( + filter: CheckpointFilter = CheckpointFilter.empty, + includeProperties: Boolean = false, + pageSize: Int = 100 + ): F[RequestResult[Seq[CheckpointV2DTO]]] = { + for { + partitioningIdOrError <- partitioningId(partitioning) + checkpointsOrError <- mapRequestResultF( + partitioningIdOrError, + (partitioningId: Long) => + queryAllPages(pageSize, queryCheckpoints(partitioningId, filter, _, _, includeProperties)) ) } yield checkpointsOrError } @@ -69,7 +97,7 @@ case class PartitioningReader[F[_]](partitioning: AtumPartitions)(implicit * Function to retrieve a page of checkpoints of the given name belonging to the partitioning. (While the usual logic * would suggest, there would be only one checkpoint of a name, nothing prevents to have checkpoints of the same name; * also during reprocessing the checkpoints might multiply.) - * The checkpoints are ordered by their creation order. + * The checkpoints are ordered by their process start time, latest first. * * @param checkpointName - the name to filter with * @param pageSize - the size of the page (record count) to be returned @@ -77,25 +105,20 @@ case class PartitioningReader[F[_]](partitioning: AtumPartitions)(implicit * @param includeProperties - whether to include checkpoint properties in the response * @return - a page of checkpoints */ + @deprecated("Use getCheckpointsPage with filter = CheckpointFilter(name = Some(checkpointName))", "0.9.0") def getCheckpointsOfNamePage( checkpointName: String, pageSize: Int = 10, offset: Long = 0, includeProperties: Boolean = false ): F[RequestResult[PaginatedResponse[CheckpointV2DTO]]] = { - for { - partitioningIdOrError <- partitioningId(partitioning) - checkpointsOrError <- mapRequestResultF( - partitioningIdOrError, - queryCheckpoints(_, Some(checkpointName), Map.empty, pageSize, offset, includeProperties) - ) - } yield checkpointsOrError + getCheckpointsPage(pageSize, offset, includeProperties, CheckpointFilter(name = Some(checkpointName))) } /** * Function to retrieve a page of checkpoints belonging to the partitioning that have all the given * checkpoint properties (matching both property name and value). - * The checkpoints are ordered by their creation order. + * The checkpoints are ordered by their process start time, latest first. * * @param checkpointProperties - the checkpoint properties (key-value pairs) to filter with; * a checkpoint is returned only if it has all of them @@ -104,19 +127,15 @@ case class PartitioningReader[F[_]](partitioning: AtumPartitions)(implicit * @param includeProperties - whether to include checkpoint properties in the response * @return - a page of checkpoints */ + @deprecated("Use getCheckpointsPage with filter = CheckpointFilter(properties = ...)", "0.9.0") def getCheckpointsByPropertiesPage( checkpointProperties: Map[String, String], pageSize: Int = 10, offset: Long = 0, includeProperties: Boolean = false ): F[RequestResult[PaginatedResponse[CheckpointV2DTO]]] = { - for { - partitioningIdOrError <- partitioningId(partitioning) - checkpointsOrError <- mapRequestResultF( - partitioningIdOrError, - queryCheckpoints(_, None, checkpointProperties, pageSize, offset, includeProperties) - ) - } yield checkpointsOrError + val properties = checkpointProperties.map { case (propertyName, value) => propertyName -> Set(value) } + getCheckpointsPage(pageSize, offset, includeProperties, CheckpointFilter(properties = properties)) } /** @@ -150,23 +169,17 @@ case class PartitioningReader[F[_]](partitioning: AtumPartitions)(implicit private def queryCheckpoints( partitioningId: Long, - checkpointName: Option[String], - checkpointProperties: Map[String, String], + filter: CheckpointFilter, limit: Int, offset: Long, includeProperties: Boolean ): F[RequestResult[PaginatedResponse[CheckpointV2DTO]]] = { val endpoint = s"/$Api/$V2/${V2Paths.Partitionings}/$partitioningId/${V2Paths.Checkpoints}" - val propertiesParam: Option[(String, String)] = - if (checkpointProperties.isEmpty) None - else Some(QueryParamNames.CheckpointProperties -> checkpointProperties.asBase64EncodedJsonString) val params = Map( QueryParamNames.Limit -> limit.toString, QueryParamNames.Offset -> offset.toString, QueryParamNames.IncludeProperties -> includeProperties.toString - ) ++ - checkpointName.map(QueryParamNames.CheckpointName -> _) ++ - propertiesParam + ) ++ filter.toQueryParams getQuery(endpoint, params) } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala b/reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala index 46b80ebd9..393312fb9 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala @@ -22,6 +22,7 @@ import sttp.client3.circe.asJson import sttp.model.Uri import sttp.monad.MonadError import sttp.monad.syntax._ +import za.co.absa.atum.model.envelopes.SuccessResponse.PaginatedResponse import za.co.absa.atum.reader.core.RequestResult._ import za.co.absa.atum.reader.server.ServerConfig import za.co.absa.atum.reader.exceptions.RequestException.CirceError @@ -48,6 +49,27 @@ abstract class Reader[F[_]](implicit case Left(a) => me.unit(Left(a)) } + /** + * Queries the pages one after another, starting at offset 0, until the server reports there is no more data. + * + * @param pageSize - the size of the page (record count) to query + * @param queryPage - function querying a page of the given size (limit) at the given offset + * @return - the records of all the pages, in order, or the first error encountered + */ + protected def queryAllPages[T]( + pageSize: Int, + queryPage: (Int, Long) => F[RequestResult[PaginatedResponse[T]]] + ): F[RequestResult[Seq[T]]] = { + def queryFrom(offset: Long, collected: Vector[T]): F[RequestResult[Seq[T]]] = { + queryPage(pageSize, offset).flatMap { + case Right(page) if page.pagination.hasMore => queryFrom(offset + pageSize, collected ++ page.data) + case Right(page) => me.unit(RequestOK[Seq[T]](collected ++ page.data)) + case Left(error) => me.unit(RequestFail[Seq[T]](error)) + } + } + queryFrom(0, Vector.empty) + } + protected def getQuery[R: Decoder]( endpointUri: String, params: Map[String, String] = Map.empty diff --git a/reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala b/reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala new file mode 100644 index 000000000..33b8e93c2 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala @@ -0,0 +1,78 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.requests + +import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ + +import java.time.ZonedDateTime + +/** + * Filter of a checkpoints query. All its parts are optional and a checkpoint has to satisfy all the given ones. + * + * Example - checkpoints of two executions processed in the last two months, latest first: + * {{{ + * CheckpointFilter( + * properties = Map("executionID" -> Set("a", "b")), + * from = Some(ZonedDateTime.now().minusMonths(2)) + * ) + * }}} + * + * Note: the time window and properties with more than one value require an Atum server supporting them; older servers + * ignore the time window and reject multi-value properties. + * + * @param name - only checkpoints of this name + * @param properties - only checkpoints that, for every property name given, have that property with one of the given + * values, e.g. `Map("executionID" -> Set("a", "b"))` means `executionID IN (a, b)` + * @param from - only checkpoints with process start time at or after this time (inclusive) + * @param to - only checkpoints with process start time before this time (exclusive) + * @param latestFirst - order of the checkpoints by their process start time: latest first if `true`, earliest first if + * `false`, server default (latest first) if not specified + */ +case class CheckpointFilter( + name: Option[String] = None, + properties: Map[String, Set[String]] = Map.empty, + from: Option[ZonedDateTime] = None, + to: Option[ZonedDateTime] = None, + latestFirst: Option[Boolean] = None +) { + + private[reader] def toQueryParams: Map[String, String] = { + name.map(QueryParamNames.CheckpointName -> _).toMap ++ + propertiesParam ++ + from.map(QueryParamNames.From -> _.toInstant.toString) ++ + to.map(QueryParamNames.To -> _.toInstant.toString) ++ + latestFirst.map(QueryParamNames.LatestFirst -> _.toString) + } + + private def propertiesParam: Option[(String, String)] = { + if (properties.isEmpty) { + None + } else if (properties.values.forall(_.size == 1)) { + // single values are sent in the original single-value format, understood by older servers too + val singleValues = properties.map { case (propertyName, values) => propertyName -> values.head } + Some(QueryParamNames.CheckpointProperties -> singleValues.asBase64EncodedJsonString) + } else { + val multiValues = properties.map { case (propertyName, values) => propertyName -> values.toSeq.sorted } + Some(QueryParamNames.CheckpointProperties -> multiValues.asBase64EncodedJsonString) + } + } + +} + +object CheckpointFilter { + val empty: CheckpointFilter = CheckpointFilter() +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala b/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala index f1bf111f9..e17baddcd 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala @@ -22,4 +22,7 @@ object QueryParamNames { final val CheckpointName = "checkpoint-name" final val CheckpointProperties = "checkpoint-properties" final val IncludeProperties = "include-properties" + final val LatestFirst = "latest-first" + final val From = "from" + final val To = "to" } diff --git a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala index b30611050..df0ad5b49 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala @@ -21,6 +21,7 @@ import sttp.capabilities import sttp.client3.monad.IdMonad import sttp.client3.{Identity, Response, SttpBackend} import sttp.client3.testing.SttpBackendStub +import sttp.model.StatusCode import sttp.model.Uri.QuerySegment.KeyValue import sttp.monad.MonadError import za.co.absa.atum.model.ResultValueType @@ -31,10 +32,12 @@ import za.co.absa.atum.model.envelopes.SuccessResponse.PaginatedResponse import za.co.absa.atum.model.types.basic.{AtumPartitions, AtumPartitionsOps} import za.co.absa.atum.reader.FlowReaderUnitTests._ import za.co.absa.atum.reader.PartitioningReaderUnitTests.checkpointsResponseWithProperties +import za.co.absa.atum.reader.requests.CheckpointFilter import za.co.absa.atum.reader.server.ServerConfig import java.time.ZonedDateTime import java.util.UUID +import scala.annotation.nowarn class FlowReaderUnitTests extends AnyFunSuiteLike { private implicit val serverConfig: ServerConfig = ServerConfig.fromConfig() @@ -192,7 +195,7 @@ class FlowReaderUnitTests extends AnyFunSuiteLike { ) val reader = FlowReader(atumPartitions) - val result = reader.getCheckpointsOfNamePage("Test checkpoints 1") + val result = (reader.getCheckpointsOfNamePage("Test checkpoints 1"): @nowarn("cat=deprecation")) assert(result == Right(expectedData)) } @@ -273,7 +276,7 @@ class FlowReaderUnitTests extends AnyFunSuiteLike { ) val reader = FlowReader(atumPartitions) - val result = reader.getCheckpointsOfNamePage("Test checkpoints 1", includeProperties = true) + val result = (reader.getCheckpointsOfNamePage("Test checkpoints 1", includeProperties = true): @nowarn("cat=deprecation")) assert(result == Right(expectedData)) } @@ -299,10 +302,90 @@ class FlowReaderUnitTests extends AnyFunSuiteLike { val reader = FlowReader(atumPartitions) // base64url-encoded {"executionID":"019f8981-7868-79fc-81d3-8143a4706f8a"} - val result = reader.getCheckpointsByPropertiesPage(Map("executionID" -> "019f8981-7868-79fc-81d3-8143a4706f8a")) + val result = ( + reader.getCheckpointsByPropertiesPage(Map("executionID" -> "019f8981-7868-79fc-81d3-8143a4706f8a")): @nowarn("cat=deprecation") + ) assert(result.isRight) } + test("The flow checkpoints are queried with name, multi-value properties and time window in a single request") { + implicit val server: SttpBackendStub[Identity, capabilities.WebSockets] = SttpBackendStub.synchronous + .whenRequestMatchesPartial { + case r if r.uri.path.endsWith(List("partitionings")) => + Response.ok(partitioningResponse) + case r if r.uri.path.endsWith(List("partitionings", "7", "main-flow")) => + Response.ok(flowResponse) + case r if r.uri.path.endsWith(List("flows", "42", "checkpoints")) => + assert(r.uri.querySegments.contains(KeyValue("limit", "5"))) + assert(r.uri.querySegments.contains(KeyValue("offset", "10"))) + assert(r.uri.querySegments.contains(KeyValue("include-properties", "true"))) + assert(r.uri.querySegments.contains(KeyValue("checkpoint-name", "Test checkpoints 1"))) + assert(r.uri.querySegments.contains(KeyValue("checkpoint-properties", multiValuePropertiesEncoded))) + assert(r.uri.querySegments.contains(KeyValue("from", "2026-06-01T00:00:00Z"))) + assert(r.uri.querySegments.contains(KeyValue("to", "2026-08-01T00:00:00Z"))) + assert(r.uri.querySegments.contains(KeyValue("latest-first", "false"))) + Response.ok(checkpointsResponse) + } + + val filter = CheckpointFilter( + name = Some("Test checkpoints 1"), + properties = Map("executionID" -> Set("id2", "id1")), + from = Some(ZonedDateTime.parse("2026-06-01T02:00:00+02:00")), + to = Some(ZonedDateTime.parse("2026-08-01T00:00:00Z")), + latestFirst = Some(false) + ) + val result = FlowReader(AtumPartitions(List("a" -> "b", "c" -> "d"))) + .getCheckpointsPage(pageSize = 5, offset = 10, includeProperties = true, filter = filter) + assert(result.map(_.data.size) == Right(2)) + } + + test("All flow checkpoints satisfying the filter are queried page by page") { + var flowIdQueries = 0 + var queriedOffsets = Vector.empty[String] + implicit val server: SttpBackendStub[Identity, capabilities.WebSockets] = SttpBackendStub.synchronous + .whenRequestMatchesPartial { + case r if r.uri.path.endsWith(List("partitionings")) => + Response.ok(partitioningResponse) + case r if r.uri.path.endsWith(List("partitionings", "7", "main-flow")) => + flowIdQueries += 1 + Response.ok(flowResponse) + case r if r.uri.path.endsWith(List("flows", "42", "checkpoints")) => + assert(r.uri.querySegments.contains(KeyValue("limit", "2"))) + assert(r.uri.querySegments.contains(KeyValue("from", "2026-06-01T00:00:00Z"))) + val offset = r.uri.paramsMap("offset") + queriedOffsets :+= offset + if (offset == "0") Response.ok(checkpointsResponseHasMore) else Response.ok(checkpointsResponse) + } + + val filter = CheckpointFilter(from = Some(ZonedDateTime.parse("2026-06-01T00:00:00Z"))) + val result = FlowReader(AtumPartitions(List("a" -> "b", "c" -> "d"))).getAllCheckpoints(filter, pageSize = 2) + + assert(result.map(_.size) == Right(4)) + assert(queriedOffsets == Vector("0", "2")) + assert(flowIdQueries == 1) + } + + test("Querying all flow checkpoints stops at the first failing page") { + var queriedOffsets = Vector.empty[String] + implicit val server: SttpBackendStub[Identity, capabilities.WebSockets] = SttpBackendStub.synchronous + .whenRequestMatchesPartial { + case r if r.uri.path.endsWith(List("partitionings")) => + Response.ok(partitioningResponse) + case r if r.uri.path.endsWith(List("partitionings", "7", "main-flow")) => + Response.ok(flowResponse) + case r if r.uri.path.endsWith(List("flows", "42", "checkpoints")) => + val offset = r.uri.paramsMap("offset") + queriedOffsets :+= offset + if (offset == "0") Response.ok(checkpointsResponseHasMore) + else Response(internalServerErrorResponse, StatusCode.InternalServerError) + } + + val result = FlowReader(AtumPartitions(List("a" -> "b", "c" -> "d"))).getAllCheckpoints(pageSize = 2) + + assert(result.isLeft) + assert(queriedOffsets == Vector("0", "2")) + } + test("Instantiate FlowReader with implicit arguments passed explicitly") { val atumPartitions: AtumPartitions = AtumPartitions(List( "a" -> "b", @@ -336,6 +419,12 @@ class FlowReaderUnitTests extends AnyFunSuiteLike { object FlowReaderUnitTests { + // base64url-encoded {"executionID":["id1","id2"]} + private val multiValuePropertiesEncoded = "eyJleGVjdXRpb25JRCI6WyJpZDEiLCJpZDIiXX0=" + + private val internalServerErrorResponse = + """{"message": "boom", "requestId": "29ce91a7-b668-41d2-a160-26402551fb0b"}""" + private val partitioningEncoded = "W3sia2V5IjoiYSIsInZhbHVlIjoiYiJ9LHsia2V5IjoiYyIsInZhbHVlIjoiZCJ9XQ==" private val partitioningResponse = @@ -453,6 +542,8 @@ object FlowReaderUnitTests { |} |""".stripMargin + private val checkpointsResponseHasMore = checkpointsResponse.replace("\"hasMore\" : false", "\"hasMore\" : true") + private val checkpointsResponseWithProperties = """ |{ diff --git a/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala index 37745ed07..ceb25edb6 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala @@ -39,10 +39,12 @@ import za.co.absa.atum.model.envelopes.Pagination import za.co.absa.atum.model.envelopes.SuccessResponse.PaginatedResponse import za.co.absa.atum.model.types.basic.AtumPartitions import za.co.absa.atum.reader.PartitioningReaderUnitTests._ +import za.co.absa.atum.reader.requests.CheckpointFilter import za.co.absa.atum.reader.server.ServerConfig import java.time.ZonedDateTime import java.util.UUID +import scala.annotation.nowarn class PartitioningReaderUnitTests extends AnyFunSuiteLike { private implicit val serverConfig: ServerConfig = ServerConfig.fromConfig() @@ -243,7 +245,7 @@ class PartitioningReaderUnitTests extends AnyFunSuiteLike { ) val reader = PartitioningReader(atumPartitions) - val result = reader.getCheckpointsOfNamePage("Test checkpoints 1") + val result = (reader.getCheckpointsOfNamePage("Test checkpoints 1"): @nowarn("cat=deprecation")) assert(result == Right(expected)) } @@ -269,10 +271,60 @@ class PartitioningReaderUnitTests extends AnyFunSuiteLike { val reader = PartitioningReader(atumPartitions) // base64url-encoded {"executionID":"019f8981-7868-79fc-81d3-8143a4706f8a"} - val result = reader.getCheckpointsByPropertiesPage(Map("executionID" -> "019f8981-7868-79fc-81d3-8143a4706f8a")) + val result = ( + reader.getCheckpointsByPropertiesPage(Map("executionID" -> "019f8981-7868-79fc-81d3-8143a4706f8a")): @nowarn("cat=deprecation") + ) assert(result.isRight) } + test("The partitioning checkpoints are queried with name, multi-value properties and time window in a single request") { + implicit val server: SttpBackendStub[Identity, capabilities.WebSockets] = SttpBackendStub.synchronous + .whenRequestMatchesPartial { + case r if r.uri.path.endsWith(List(V2Paths.Partitionings)) => + Response.ok(partitioningResponse) + case r if r.uri.path.endsWith(List(V2Paths.Partitionings, "7", V2Paths.Checkpoints)) => + assert(r.uri.querySegments.contains(KeyValue("limit", "10"))) + assert(r.uri.querySegments.contains(KeyValue("offset", "0"))) + assert(r.uri.querySegments.contains(KeyValue("checkpoint-name", "Test checkpoints 1"))) + // base64url-encoded {"executionID":["id1","id2"]} + assert(r.uri.querySegments.contains(KeyValue("checkpoint-properties", "eyJleGVjdXRpb25JRCI6WyJpZDEiLCJpZDIiXX0="))) + assert(r.uri.querySegments.contains(KeyValue("from", "2026-06-01T00:00:00Z"))) + assert(!r.uri.paramsMap.contains("to")) + assert(!r.uri.paramsMap.contains("latest-first")) + Response.ok(checkpointsResponse) + } + + val filter = CheckpointFilter( + name = Some("Test checkpoints 1"), + properties = Map("executionID" -> Set("id1", "id2")), + from = Some(ZonedDateTime.parse("2026-06-01T00:00:00Z")) + ) + val result = PartitioningReader(AtumPartitions(List("a" -> "b", "c" -> "d"))).getCheckpointsPage(filter = filter) + assert(result.map(_.data.size) == Right(2)) + } + + test("All partitioning checkpoints satisfying the filter are queried page by page") { + var queriedOffsets = Vector.empty[String] + implicit val server: SttpBackendStub[Identity, capabilities.WebSockets] = SttpBackendStub.synchronous + .whenRequestMatchesPartial { + case r if r.uri.path.endsWith(List(V2Paths.Partitionings)) => + Response.ok(partitioningResponse) + case r if r.uri.path.endsWith(List(V2Paths.Partitionings, "7", V2Paths.Checkpoints)) => + assert(r.uri.querySegments.contains(KeyValue("limit", "2"))) + assert(r.uri.querySegments.contains(KeyValue("checkpoint-name", "Test checkpoints 1"))) + val offset = r.uri.paramsMap("offset") + queriedOffsets :+= offset + if (offset == "4") Response.ok(checkpointsResponse) + else Response.ok(checkpointsResponse.replace("\"hasMore\" : false", "\"hasMore\" : true")) + } + + val result = PartitioningReader(AtumPartitions(List("a" -> "b", "c" -> "d"))) + .getAllCheckpoints(CheckpointFilter(name = Some("Test checkpoints 1")), pageSize = 2) + + assert(result.map(_.size) == Right(6)) + assert(queriedOffsets == Vector("0", "2", "4")) + } + test("The partitioning additional data are properly queried and delivered as DTO") { val atumPartitions: AtumPartitions = AtumPartitions( List( diff --git a/reader/src/test/scala/za/co/absa/atum/reader/requests/CheckpointFilterUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/requests/CheckpointFilterUnitTests.scala new file mode 100644 index 000000000..e40635059 --- /dev/null +++ b/reader/src/test/scala/za/co/absa/atum/reader/requests/CheckpointFilterUnitTests.scala @@ -0,0 +1,65 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.requests + +import org.scalatest.funsuite.AnyFunSuiteLike +import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ + +import java.time.ZonedDateTime + +class CheckpointFilterUnitTests extends AnyFunSuiteLike { + + test("An empty filter adds no query params") { + assert(CheckpointFilter.empty.toQueryParams.isEmpty) + } + + test("Single-value properties are sent in the single-value format understood by older servers too") { + val filter = CheckpointFilter(properties = Map("executionID" -> Set("019f8981-7868-79fc-81d3-8143a4706f8a"))) + // base64url-encoded {"executionID":"019f8981-7868-79fc-81d3-8143a4706f8a"} + assert( + filter.toQueryParams == Map( + "checkpoint-properties" -> "eyJleGVjdXRpb25JRCI6IjAxOWY4OTgxLTc4NjgtNzlmYy04MWQzLTgxNDNhNDcwNmY4YSJ9" + ) + ) + } + + test("Multi-value properties are sent as arrays of values, all properties in the same format") { + val filter = CheckpointFilter(properties = Map("executionID" -> Set("b", "a"), "env" -> Set("prod"))) + val encoded = filter.toQueryParams("checkpoint-properties") + assert( + encoded.fromBase64As[Map[String, Seq[String]]] == Right(Map("executionID" -> Seq("a", "b"), "env" -> Seq("prod"))) + ) + } + + test("The time window is sent as UTC instants and all filter parts are combined") { + val filter = CheckpointFilter( + name = Some("checkpoint name"), + from = Some(ZonedDateTime.parse("2026-06-01T02:00:00+02:00[Europe/Budapest]")), + to = Some(ZonedDateTime.parse("2026-08-01T00:00:00.5Z")), + latestFirst = Some(false) + ) + assert( + filter.toQueryParams == Map( + "checkpoint-name" -> "checkpoint name", + "from" -> "2026-06-01T00:00:00Z", + "to" -> "2026-08-01T00:00:00.500Z", + "latest-first" -> "false" + ) + ) + } + +} diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala index 2534988f7..b04dc1f8b 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala @@ -35,6 +35,8 @@ import doobie.postgres.implicits._ import za.co.absa.atum.server.model.database.CheckpointItemWithPartitioningFromDB +import java.time.ZonedDateTime + class GetFlowCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieMultipleResultFunctionWithAggStatus[GetFlowCheckpointsArgs, Option[ CheckpointItemWithPartitioningFromDB @@ -44,7 +46,10 @@ class GetFlowCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task] fr"${input.limit}", fr"${input.offset}", fr"${input.checkpointName}", - fr"${input.checkpointProperties.map(_.asJson)}" + fr"${input.checkpointProperties.map(_.asJson)}", + fr"${input.latestFirst}", + fr"${input.from}", + fr"${input.to}" ) ) with StandardStatusHandling @@ -73,7 +78,10 @@ object GetFlowCheckpoints { limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, Seq[String]]] + checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime] ) val layer: URLayer[PostgresDatabaseProvider, GetFlowCheckpoints] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala index b43ec3662..0632ac744 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala @@ -35,6 +35,8 @@ import io.circe.generic.auto._ import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstRowStatusAggregator import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling +import java.time.ZonedDateTime + class GetPartitioningCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieMultipleResultFunctionWithAggStatus[GetPartitioningCheckpointsArgs, Option[ CheckpointItemFromDB.Paginated @@ -45,7 +47,9 @@ class GetPartitioningCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngi fr"${args.offset}", fr"${args.checkpointName}", fr"${args.checkpointProperties.map(_.asJson)}", - fr"${args.latestFirst}" + fr"${args.latestFirst}", + fr"${args.from}", + fr"${args.to}" ) ) with StandardStatusHandling @@ -72,7 +76,9 @@ object GetPartitioningCheckpoints { offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], - latestFirst: Option[Boolean] + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime] ) val layer: URLayer[PostgresDatabaseProvider, GetPartitioningCheckpoints] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointController.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointController.scala index 428f55a1e..4d521b999 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointController.scala @@ -22,6 +22,7 @@ import za.co.absa.atum.model.envelopes.SuccessResponse.{PaginatedResponse, Singl import zio.IO import zio.macros.accessible +import java.time.ZonedDateTime import java.util.UUID @accessible @@ -45,6 +46,8 @@ trait CheckpointController { checkpointName: Option[String] = None, checkpointProperties: Option[Map[String, Seq[String]]] = None, latestFirst: Option[Boolean] = None, + from: Option[ZonedDateTime] = None, + to: Option[ZonedDateTime] = None, includeProperties: Boolean ): IO[ErrorResponse, PaginatedResponse[CheckpointV2DTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerImpl.scala index 46897914e..464cf05ec 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerImpl.scala @@ -25,6 +25,7 @@ import za.co.absa.atum.server.api.v2.service.CheckpointService import za.co.absa.atum.server.model.PaginatedResult import zio._ +import java.time.ZonedDateTime import java.util.UUID class CheckpointControllerImpl(checkpointService: CheckpointService) extends CheckpointController with BaseController { @@ -66,6 +67,8 @@ class CheckpointControllerImpl(checkpointService: CheckpointService) extends Che checkpointName: Option[String] = None, checkpointProperties: Option[Map[String, Seq[String]]] = None, latestFirst: Option[Boolean] = None, + from: Option[ZonedDateTime] = None, + to: Option[ZonedDateTime] = None, includeProperties: Boolean ): IO[ErrorResponse, PaginatedResponse[CheckpointV2DTO]] = { mapToPaginatedResponse( @@ -79,6 +82,8 @@ class CheckpointControllerImpl(checkpointService: CheckpointService) extends Che checkpointName, checkpointProperties, latestFirst, + from, + to, includeProperties ) ) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowController.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowController.scala index 23be150c6..90cca4761 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowController.scala @@ -22,6 +22,8 @@ import za.co.absa.atum.model.envelopes.SuccessResponse.PaginatedResponse import zio.IO import zio.macros.accessible +import java.time.ZonedDateTime + @accessible trait FlowController { @@ -31,6 +33,9 @@ trait FlowController { offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ErrorResponse, PaginatedResponse[CheckpointWithPartitioningDTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerImpl.scala index 8a031b0f4..3d1419de0 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerImpl.scala @@ -24,6 +24,8 @@ import za.co.absa.atum.server.api.v2.service.FlowService import za.co.absa.atum.server.model.PaginatedResult import zio._ +import java.time.ZonedDateTime + class FlowControllerImpl(flowService: FlowService) extends FlowController with BaseController { // to be replaced (and moved to checkpointcontroller) with new implementation in #233 @@ -33,11 +35,24 @@ class FlowControllerImpl(flowService: FlowService) extends FlowController with B offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ErrorResponse, PaginatedResponse[CheckpointWithPartitioningDTO]] = { val flowData = serviceCall[PaginatedResult[CheckpointWithPartitioningDTO], PaginatedResult[CheckpointWithPartitioningDTO]]( - flowService.getFlowCheckpoints(flowId, limit, offset, checkpointName, checkpointProperties, includeProperties) + flowService.getFlowCheckpoints( + flowId, + limit, + offset, + checkpointName, + checkpointProperties, + latestFirst, + from, + to, + includeProperties + ) ) mapToPaginatedResponse(limit, offset, flowData) } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala index 906a22cc0..99baf6be9 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala @@ -20,7 +20,7 @@ import sttp.model.StatusCode import sttp.tapir.generic.auto.schemaForCaseClass import sttp.tapir.json.circe.jsonBody import sttp.tapir.ztapir._ -import sttp.tapir.{Codec, CodecFormat, DecodeResult, PublicEndpoint, Validator} +import sttp.tapir.{Codec, CodecFormat, DecodeResult, EndpointInput, PublicEndpoint, ValidationResult, Validator} import za.co.absa.atum.model.ApiPaths._ import za.co.absa.atum.model.dto._ import za.co.absa.atum.model.envelopes.ErrorResponse @@ -29,6 +29,7 @@ import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ import za.co.absa.atum.server.api.v2.controller.{CheckpointController, FlowController, PartitioningController} import za.co.absa.atum.server.api.common.http.{BaseEndpoints, HttpEnv} +import java.time.ZonedDateTime import java.util.UUID import scala.util.{Failure, Success, Try} @@ -49,6 +50,21 @@ object Endpoints extends BaseEndpoints { } }(_.asBase64EncodedJsonString) + // Optional half-open window `from <= process start time < to` for checkpoint queries, supplied as ISO-8601 date-times + // with an offset, e.g. `from=2026-06-01T00:00:00Z`. A malformed value, or `from` not before `to`, results in a + // 400 Bad Request. + private val checkpointTimeWindowInput: EndpointInput[(Option[ZonedDateTime], Option[ZonedDateTime])] = + query[Option[ZonedDateTime]]("from") + .description("inclusive lower bound of the checkpoint's process start time (ISO-8601 date-time with offset)") + .and( + query[Option[ZonedDateTime]]("to") + .description("exclusive upper bound of the checkpoint's process start time (ISO-8601 date-time with offset)") + ) + .validate(Validator.custom { + case (Some(from), Some(to)) if !from.isBefore(to) => ValidationResult.Invalid("'from' must be before 'to'") + case _ => ValidationResult.Valid + }) + val postCheckpointEndpoint : PublicEndpoint[(Long, CheckpointV2DTO), ErrorResponse, (SingleSuccessResponse[CheckpointV2DTO], String), Any] = { apiV2.post @@ -117,7 +133,7 @@ object Endpoints extends BaseEndpoints { } val getPartitioningCheckpointsEndpoint - : PublicEndpoint[(Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Boolean), ErrorResponse, PaginatedResponse[ + : PublicEndpoint[(Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Option[ZonedDateTime], Option[ZonedDateTime], Boolean), ErrorResponse, PaginatedResponse[ CheckpointV2DTO ], Any] = { apiV2.get @@ -128,6 +144,7 @@ object Endpoints extends BaseEndpoints { .in(query[Option[String]]("checkpoint-name")) .in(query[Option[Map[String, Seq[String]]]]("checkpoint-properties")) .in(query[Option[Boolean]]("latest-first")) + .in(checkpointTimeWindowInput) .in(query[Boolean]("include-properties").default(false)) .out(statusCode(StatusCode.Ok)) .out(jsonBody[PaginatedResponse[CheckpointV2DTO]]) @@ -135,7 +152,7 @@ object Endpoints extends BaseEndpoints { } val getFlowCheckpointsEndpoint - : PublicEndpoint[(Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Boolean), ErrorResponse, PaginatedResponse[ + : PublicEndpoint[(Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Option[ZonedDateTime], Option[ZonedDateTime], Boolean), ErrorResponse, PaginatedResponse[ CheckpointWithPartitioningDTO ], Any] = { apiV2.get @@ -144,6 +161,8 @@ object Endpoints extends BaseEndpoints { .in(query[Long]("offset").default(0L).validate(Validator.min(0L))) .in(query[Option[String]]("checkpoint-name")) .in(query[Option[Map[String, Seq[String]]]]("checkpoint-properties")) + .in(query[Option[Boolean]]("latest-first")) + .in(checkpointTimeWindowInput) .in(query[Boolean]("include-properties").default(false)) .out(statusCode(StatusCode.Ok)) .out(jsonBody[PaginatedResponse[CheckpointWithPartitioningDTO]]) @@ -252,23 +271,23 @@ object Endpoints extends BaseEndpoints { } ), createServerEndpoint[ - (Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Boolean), + (Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Option[ZonedDateTime], Option[ZonedDateTime], Boolean), ErrorResponse, PaginatedResponse[CheckpointV2DTO] ]( getPartitioningCheckpointsEndpoint, - { case (partitioningId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], includeProperties: Boolean) => - CheckpointController.getPartitioningCheckpoints(partitioningId, limit, offset, checkpointName, checkpointProperties, latestFirst, includeProperties) + { case (partitioningId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], from: Option[ZonedDateTime], to: Option[ZonedDateTime], includeProperties: Boolean) => + CheckpointController.getPartitioningCheckpoints(partitioningId, limit, offset, checkpointName, checkpointProperties, latestFirst, from, to, includeProperties) } ), createServerEndpoint[ - (Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Boolean), + (Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Option[ZonedDateTime], Option[ZonedDateTime], Boolean), ErrorResponse, PaginatedResponse[CheckpointWithPartitioningDTO] ]( getFlowCheckpointsEndpoint, - { case (flowId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], includeProperties: Boolean) => - FlowController.getFlowCheckpoints(flowId, limit, offset, checkpointName, checkpointProperties, includeProperties) + { case (flowId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], from: Option[ZonedDateTime], to: Option[ZonedDateTime], includeProperties: Boolean) => + FlowController.getFlowCheckpoints(flowId, limit, offset, checkpointName, checkpointProperties, latestFirst, from, to, includeProperties) } ), createServerEndpoint(getPartitioningByIdEndpoint, PartitioningController.getPartitioningById), diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepository.scala index 14856ecf2..018cf3df6 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepository.scala @@ -22,6 +22,7 @@ import za.co.absa.atum.server.model.PaginatedResult import zio._ import zio.macros.accessible +import java.time.ZonedDateTime import java.util.UUID @accessible @@ -39,6 +40,8 @@ trait CheckpointRepository { checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[DatabaseError, PaginatedResult[CheckpointV2DTO]] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryImpl.scala index 3a8cb1c22..d1c13b425 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryImpl.scala @@ -31,6 +31,7 @@ import za.co.absa.atum.server.model.database.CheckpointItemFromDB import zio._ import zio.interop.catz.asyncInstance +import java.time.ZonedDateTime import java.util.UUID class CheckpointRepositoryImpl( @@ -77,11 +78,22 @@ class CheckpointRepositoryImpl( checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[DatabaseError, PaginatedResult[CheckpointV2DTO]] = { dbMultipleResultCallWithAggregatedStatus( getPartitioningCheckpointsFn( - GetPartitioningCheckpointsArgs(partitioningId, limit, offset, checkpointName, checkpointProperties, latestFirst) + GetPartitioningCheckpointsArgs( + partitioningId, + limit, + offset, + checkpointName, + checkpointProperties, + latestFirst, + from, + to + ) ), "getPartitioningCheckpoints" ) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepository.scala index d47df8ff1..15e9b9951 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepository.scala @@ -22,6 +22,8 @@ import za.co.absa.atum.server.model.PaginatedResult import zio._ import zio.macros.accessible +import java.time.ZonedDateTime + @accessible trait FlowRepository { @@ -31,6 +33,9 @@ trait FlowRepository { offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[DatabaseError, PaginatedResult[CheckpointWithPartitioningDTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryImpl.scala index b668384be..d861b046d 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryImpl.scala @@ -29,6 +29,8 @@ import za.co.absa.atum.server.model.database.CheckpointItemWithPartitioningFromD import zio._ import zio.interop.catz.asyncInstance +import java.time.ZonedDateTime + class FlowRepositoryImpl( getFlowCheckpointsFn: GetFlowCheckpoints, override val getCheckpointPropertiesFn: GetCheckpointProperties @@ -40,10 +42,15 @@ class FlowRepositoryImpl( offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[DatabaseError, PaginatedResult[CheckpointWithPartitioningDTO]] = { dbMultipleResultCallWithAggregatedStatus( - getFlowCheckpointsFn(GetFlowCheckpointsArgs(flowId, limit, offset, checkpointName, checkpointProperties)), + getFlowCheckpointsFn( + GetFlowCheckpointsArgs(flowId, limit, offset, checkpointName, checkpointProperties, latestFirst, from, to) + ), "getFlowCheckpoints" ) .map(_.flatten) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointService.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointService.scala index 4e6d4439e..c59ca56cd 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointService.scala @@ -22,6 +22,7 @@ import za.co.absa.atum.server.model.PaginatedResult import zio.IO import zio.macros.accessible +import java.time.ZonedDateTime import java.util.UUID @accessible @@ -42,6 +43,8 @@ trait CheckpointService { checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ServiceError, PaginatedResult[CheckpointV2DTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceImpl.scala index 05428e894..a53b57285 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceImpl.scala @@ -23,6 +23,7 @@ import za.co.absa.atum.server.api.v2.repository.CheckpointRepository import za.co.absa.atum.server.model.PaginatedResult import zio._ +import java.time.ZonedDateTime import java.util.UUID class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends CheckpointService with BaseService { @@ -52,6 +53,8 @@ class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ServiceError, PaginatedResult[CheckpointV2DTO]] = { repositoryCall( @@ -62,6 +65,8 @@ class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends checkpointName, checkpointProperties, latestFirst, + from, + to, includeProperties ), "getPartitioningCheckpoints" diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowService.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowService.scala index 27e0bbe5c..5563086b9 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowService.scala @@ -22,6 +22,8 @@ import za.co.absa.atum.server.model.PaginatedResult import zio._ import zio.macros.accessible +import java.time.ZonedDateTime + @accessible trait FlowService { @@ -31,6 +33,9 @@ trait FlowService { offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ServiceError, PaginatedResult[CheckpointWithPartitioningDTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowServiceImpl.scala index 936979a0c..555b23271 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowServiceImpl.scala @@ -23,6 +23,8 @@ import za.co.absa.atum.server.api.v2.repository.FlowRepository import za.co.absa.atum.server.model.PaginatedResult import zio._ +import java.time.ZonedDateTime + class FlowServiceImpl(flowRepository: FlowRepository) extends FlowService with BaseService { override def getFlowCheckpoints( @@ -31,10 +33,23 @@ class FlowServiceImpl(flowRepository: FlowRepository) extends FlowService with B offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ServiceError, PaginatedResult[CheckpointWithPartitioningDTO]] = { repositoryCall( - flowRepository.getFlowCheckpoints(flowId, limit, offset, checkpointName, checkpointProperties, includeProperties), + flowRepository.getFlowCheckpoints( + flowId, + limit, + offset, + checkpointName, + checkpointProperties, + latestFirst, + from, + to, + includeProperties + ), "getFlowCheckpoints" ) } diff --git a/server/src/main/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDB.scala index 25b1cf9e7..2ad4d5337 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDB.scala @@ -98,10 +98,8 @@ object CheckpointItemWithPartitioningFromDB { checkpointItems: Seq[CheckpointItemWithPartitioningFromDB] ): Either[Throwable, Seq[CheckpointWithPartitioningDTO]] = { val groupedItems = checkpointItems.groupBy(_.idCheckpoint) - val orderedCheckpointIds = checkpointItems - .sortBy(_.checkpointStartTime)(Ordering[ZonedDateTime].reverse) - .map(_.idCheckpoint) - .distinct + // the DB function returns the items already ordered (latest or earliest first, as requested), so keep that order + val orderedCheckpointIds = checkpointItems.map(_.idCheckpoint).distinct val result = orderedCheckpointIds.map { id => fromItemsToCheckpointWithPartitioningDTO(groupedItems(id)) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala index 31147df05..f2fc9a9a3 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala @@ -25,6 +25,8 @@ import zio._ import zio.test._ import zio.interop.catz.asyncInstance +import java.time.ZonedDateTime + object GetFlowCheckpointsIntegrationTests extends ConfigProviderTest { override def spec: Spec[TestEnvironment with Scope, Any] = { @@ -37,7 +39,10 @@ object GetFlowCheckpointsIntegrationTests extends ConfigProviderTest { limit = 10, offset = 0L, checkpointName = Some("TestCheckpointName"), - checkpointProperties = None + checkpointProperties = None, + latestFirst = None, + from = None, + to = None ) for { @@ -51,7 +56,27 @@ object GetFlowCheckpointsIntegrationTests extends ConfigProviderTest { limit = 10, offset = 0L, checkpointName = Some("TestCheckpointName"), - checkpointProperties = Some(Map("key1" -> Seq("value1", "value2"))) + checkpointProperties = Some(Map("key1" -> Seq("value1", "value2"))), + latestFirst = None, + from = None, + to = None + ) + + for { + getFlowCheckpoints <- ZIO.service[GetFlowCheckpoints] + result <- getFlowCheckpoints(args) + } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(42, "Flow not found")))) + }, + test("Should bind the latest-first flag and the process start time window") { + val args = GetFlowCheckpoints.GetFlowCheckpointsArgs( + flowId = 1L, + limit = 10, + offset = 0L, + checkpointName = None, + checkpointProperties = None, + latestFirst = Some(false), + from = Some(ZonedDateTime.parse("2026-06-01T00:00:00Z")), + to = Some(ZonedDateTime.parse("2026-08-01T00:00:00+02:00")) ) for { diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala index 3414b9cfb..f4e359fd1 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala @@ -26,6 +26,8 @@ import zio.interop.catz.asyncInstance import zio.{Scope, ZIO} import zio.test._ +import java.time.ZonedDateTime + object GetPartitioningCheckpointsIntegrationTests extends ConfigProviderTest { override def spec: Spec[TestEnvironment with Scope, Any] = { @@ -33,13 +35,21 @@ object GetPartitioningCheckpointsIntegrationTests extends ConfigProviderTest { test("Returns expected sequence of Checkpoints with non-existing partitioning id") { for { getPartitioningCheckpoints <- ZIO.service[GetPartitioningCheckpoints] - result <- getPartitioningCheckpoints(GetPartitioningCheckpointsArgs(0L, 10, 0L, None, None, None)) + result <- getPartitioningCheckpoints(GetPartitioningCheckpointsArgs(0L, 10, 0L, None, None, None, None, None)) } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) }, test("Should apply pagination (limit and offset) accurately with combined filters") { for { getPartitioningCheckpoints <- ZIO.service[GetPartitioningCheckpoints] - result <- getPartitioningCheckpoints(GetPartitioningCheckpointsArgs(0L, 10, 0L, Some("TestCheckpointName"), Some(Map("key1" -> Seq("value1", "value2"))), None)) + result <- getPartitioningCheckpoints(GetPartitioningCheckpointsArgs(0L, 10, 0L, Some("TestCheckpointName"), Some(Map("key1" -> Seq("value1", "value2"))), None, None, None)) + } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) + }, + test("Should bind the process start time window") { + val from = Some(ZonedDateTime.parse("2026-06-01T00:00:00Z")) + val to = Some(ZonedDateTime.parse("2026-08-01T00:00:00+02:00")) + for { + getPartitioningCheckpoints <- ZIO.service[GetPartitioningCheckpoints] + result <- getPartitioningCheckpoints(GetPartitioningCheckpointsArgs(0L, 10, 0L, None, None, Some(true), from, to)) } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) } ).provide( diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerUnitTests.scala index 9a846d9d5..b44674f70 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerUnitTests.scala @@ -51,11 +51,11 @@ object CheckpointControllerUnitTests extends ConfigProviderTest with TestData { when(checkpointServiceMock.getCheckpoint(partitioningId1, checkpointV2DTO3.id, includeProperties = false)) .thenReturn(ZIO.fail(GeneralServiceError("boom!"))) - when(checkpointServiceMock.getPartitioningCheckpoints(partitioningId1, 10, 0L, None, None, None, includeProperties = false)) + when(checkpointServiceMock.getPartitioningCheckpoints(partitioningId1, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultHasMore(Seq(checkpointV2DTO1)))) - when(checkpointServiceMock.getPartitioningCheckpoints(partitioningId2, 10, 0L, None, None, None, includeProperties = false)) + when(checkpointServiceMock.getPartitioningCheckpoints(partitioningId2, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultNoMore(Seq(checkpointV2DTO1)))) - when(checkpointServiceMock.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, includeProperties = false)) + when(checkpointServiceMock.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundServiceError("Partitioning not found"))) private val checkpointServiceMockLayer = ZLayer.succeed(checkpointServiceMock) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerUnitTests.scala index d78be581e..eabe38774 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerUnitTests.scala @@ -29,11 +29,11 @@ import zio.test._ object FlowControllerUnitTests extends ZIOSpecDefault with TestData { private val flowServiceMock = mock(classOf[FlowService]) - when(flowServiceMock.getFlowCheckpoints(1L, 5, 2L, None, None, includeProperties = false)) + when(flowServiceMock.getFlowCheckpoints(1L, 5, 2L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultHasMore(Seq(checkpointWithPartitioningDTO1)))) - when(flowServiceMock.getFlowCheckpoints(2L, 5, 0L, None, None, includeProperties = false)) + when(flowServiceMock.getFlowCheckpoints(2L, 5, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultNoMore(Seq(checkpointWithPartitioningDTO2)))) - when(flowServiceMock.getFlowCheckpoints(3L, 5, 0L, None, None, includeProperties = false)) + when(flowServiceMock.getFlowCheckpoints(3L, 5, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundServiceError("Flow not found"))) private val flowServiceMockLayer = ZLayer.succeed(flowServiceMock) @@ -43,20 +43,20 @@ object FlowControllerUnitTests extends ZIOSpecDefault with TestData { suite("GetFlowCheckpointsV2Suite")( test("Returns expected Seq[CheckpointV2DTO] with Pagination indicating there is more data available") { for { - result <- FlowController.getFlowCheckpoints(1L, 5, 2L, None, None, includeProperties = false) + result <- FlowController.getFlowCheckpoints(1L, 5, 2L, None, None, None, None, None, includeProperties = false) } yield assertTrue( result.data == Seq(checkpointWithPartitioningDTO1) && result.pagination == Pagination(5, 2, hasMore = true) ) }, test("Returns expected Seq[CheckpointV2DTO] with Pagination indicating there is no more data available") { for { - result <- FlowController.getFlowCheckpoints(2L, 5, 0L, None, None, includeProperties = false) + result <- FlowController.getFlowCheckpoints(2L, 5, 0L, None, None, None, None, None, includeProperties = false) } yield assertTrue( result.data == Seq(checkpointWithPartitioningDTO2) && result.pagination == Pagination(5, 0, hasMore = false) ) }, test("Returns expected NotFoundServiceError when service returns NotFoundServiceError") { - assertZIO(FlowController.getFlowCheckpoints(3L, 5, 0L, None, None, includeProperties = false).exit)( + assertZIO(FlowController.getFlowCheckpoints(3L, 5, 0L, None, None, None, None, None, includeProperties = false).exit)( failsWithA[NotFoundErrorResponse] ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala index be500cc6e..b9f72acf7 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala @@ -32,6 +32,7 @@ import zio.test.Assertion.equalTo import zio.test.{Spec, TestEnvironment, ZIOSpecDefault, assertZIO} import zio.{Scope, ZIO, ZLayer} +import java.time.ZonedDateTime import java.util.UUID object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData { @@ -45,11 +46,11 @@ object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData private val encodedExecutionIdProperties = "eyJleGVjdXRpb25JRCI6IjAxOWY4OTgxLTc4NjgtNzlmYy04MWQzLTgxNDNhNDcwNmY4YSJ9" - when(flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, None, includeProperties = false)) + when(flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn( ZIO.succeed(PaginatedResponse(Seq(checkpointWithPartitioningDTO1), Pagination(5, 0, hasMore = true), uuid)) ) - when(flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, None, includeProperties = true)) + when(flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, None, None, None, None, includeProperties = true)) .thenReturn( ZIO.succeed( PaginatedResponse( @@ -59,24 +60,56 @@ object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData ) ) ) - when(flowControllerMockV2.getFlowCheckpoints(2L, 5, 0L, None, None, includeProperties = false)) + when(flowControllerMockV2.getFlowCheckpoints(2L, 5, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn( ZIO.succeed(PaginatedResponse(Seq(checkpointWithPartitioningDTO2), Pagination(5, 0, hasMore = false), uuid)) ) - when(flowControllerMockV2.getFlowCheckpoints(3L, 5, 0L, None, None, includeProperties = false)) + when(flowControllerMockV2.getFlowCheckpoints(3L, 5, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundErrorResponse("Flow not found for a given ID"))) when( - flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, Some(executionIdProperties.view.mapValues(Seq(_)).toMap), includeProperties = false) + flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, Some(executionIdProperties.view.mapValues(Seq(_)).toMap), None, None, None, includeProperties = false) ) .thenReturn( ZIO.succeed(PaginatedResponse(Seq(checkpointWithPartitioningDTO1), Pagination(5, 0, hasMore = true), uuid)) ) + private val windowFrom = ZonedDateTime.parse("2026-06-01T00:00:00Z") + private val windowTo = ZonedDateTime.parse("2026-08-01T00:00:00Z") + + when( + flowControllerMockV2.getFlowCheckpoints( + 1L, 5, 0L, None, None, Some(false), Some(windowFrom), Some(windowTo), includeProperties = false + ) + ) + .thenReturn( + ZIO.succeed(PaginatedResponse(Seq(checkpointWithPartitioningDTO2), Pagination(5, 0, hasMore = false), uuid)) + ) + private val flowControllerMockLayerV2 = ZLayer.succeed(flowControllerMockV2) private val getFlowCheckpointServerEndpoint = Endpoints.getFlowCheckpointsEndpoint.zServerLogic({ - case (flowId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], includeProperties: Boolean) => - FlowController.getFlowCheckpoints(flowId, limit, offset, checkpointName, checkpointProperties, includeProperties) + case ( + flowId: Long, + limit: Int, + offset: Long, + checkpointName: Option[String], + checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], + includeProperties: Boolean + ) => + FlowController.getFlowCheckpoints( + flowId, + limit, + offset, + checkpointName, + checkpointProperties, + latestFirst, + from, + to, + includeProperties + ) }) def spec: Spec[TestEnvironment with Scope, Any] = { @@ -174,6 +207,47 @@ object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData ) ) }, + test("Returns an expected PaginatedResponse[CheckpointWithPartitioningDTO] when filtering by a time window, earliest first") { + val baseUri = + uri"https://test.com/api/v2/flows/1/checkpoints?limit=5&offset=0&latest-first=false&from=2026-06-01T00:00:00Z&to=2026-08-01T00:00:00Z" + val response = basicRequest + .get(baseUri) + .response(asJson[PaginatedResponse[CheckpointWithPartitioningDTO]]) + .send(backendStub) + + val body = response.map(_.body) + val statusCode = response.map(_.code) + + assertZIO(body <&> statusCode)( + equalTo( + Right(PaginatedResponse(Seq(checkpointWithPartitioningDTO2), Pagination(5, 0, hasMore = false), uuid)), + StatusCode.Ok + ) + ) + }, + test("Returns expected 400 when 'to' is not a valid date-time") { + val baseUri = uri"https://test.com/api/v2/flows/1/checkpoints?limit=5&offset=0&to=not-a-date-time" + val response = basicRequest + .get(baseUri) + .response(asJson[PaginatedResponse[CheckpointWithPartitioningDTO]]) + .send(backendStub) + + val statusCode = response.map(_.code) + + assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) + }, + test("Returns expected 400 when 'from' is after 'to'") { + val baseUri = + uri"https://test.com/api/v2/flows/1/checkpoints?limit=5&offset=0&from=2026-08-01T00:00:00Z&to=2026-06-01T00:00:00Z" + val response = basicRequest + .get(baseUri) + .response(asJson[PaginatedResponse[CheckpointWithPartitioningDTO]]) + .send(backendStub) + + val statusCode = response.map(_.code) + + assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) + }, test("Returns expected 400 when checkpoint-properties is not valid base64") { val baseUri = uri"https://test.com/api/v2/flows/1/checkpoints?limit=5&offset=0&checkpoint-properties=!!!not-base64!!!" diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala index 9f5e0d4ad..3e634a284 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala @@ -32,6 +32,7 @@ import zio.test.Assertion.equalTo import zio.test.{Spec, TestEnvironment, ZIOSpecDefault, assertZIO} import zio.{Scope, ZIO, ZLayer} +import java.time.ZonedDateTime import java.util.UUID object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData { @@ -40,9 +41,12 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T private val uuid = UUID.randomUUID() - when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, includeProperties = false)) + private val windowFrom = ZonedDateTime.parse("2026-06-01T00:00:00Z") + private val windowTo = ZonedDateTime.parse("2026-08-01T00:00:00Z") + + when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(PaginatedResponse(Seq(checkpointV2DTO1), Pagination(10, 0, hasMore = true), uuid))) - when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, includeProperties = true)) + when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, None, None, includeProperties = true)) .thenReturn( ZIO.succeed( PaginatedResponse( @@ -52,14 +56,20 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T ) ) ) - when(checkpointControllerMock.getPartitioningCheckpoints(1L, 20, 0L, None, None, None, includeProperties = false)) + when(checkpointControllerMock.getPartitioningCheckpoints(1L, 20, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(PaginatedResponse(Seq(checkpointV2DTO1), Pagination(20, 0, hasMore = false), uuid))) - when(checkpointControllerMock.getPartitioningCheckpoints(2L, 10, 0L, None, None, None, includeProperties = false)) + when(checkpointControllerMock.getPartitioningCheckpoints(2L, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundErrorResponse("partitioning not found"))) - when(checkpointControllerMock.getPartitioningCheckpoints(3L, 10, 0L, None, None, None, includeProperties = false)) + when(checkpointControllerMock.getPartitioningCheckpoints(3L, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(PaginatedResponse(Seq(checkpointV2DTO1), Pagination(10, 0, hasMore = true), uuid))) - when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, Some(Map("executionID" -> Seq("id1", "id2"))), None, includeProperties = false)) + when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, Some(Map("executionID" -> Seq("id1", "id2"))), None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(PaginatedResponse(Seq(checkpointV2DTO1), Pagination(10, 0, hasMore = true), uuid))) + when( + checkpointControllerMock.getPartitioningCheckpoints( + 1L, 10, 0L, None, None, None, Some(windowFrom), Some(windowTo), includeProperties = false + ) + ) + .thenReturn(ZIO.succeed(PaginatedResponse(Seq(checkpointV2DTO2), Pagination(10, 0, hasMore = false), uuid))) private val checkpointControllerMockLayer = ZLayer.succeed(checkpointControllerMock) @@ -73,6 +83,8 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ) => CheckpointController.getPartitioningCheckpoints( @@ -82,6 +94,8 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T checkpointName, checkpointProperties, latestFirst, + from, + to, includeProperties ) }) @@ -115,6 +129,52 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T ) ) }, + test("Returns an expected PaginatedResponse[CheckpointV2DTO] when filtering by a process start time window") { + val request = basicRequest + .get( + uri"https://test.com/api/v2/partitionings/1/checkpoints?limit=10&offset=0&from=2026-06-01T00:00:00Z&to=2026-08-01T00:00:00Z" + ) + .response(asJson[PaginatedResponse[CheckpointV2DTO]]) + + val response = request + .send(backendStub) + + val body = response.map(_.body) + val statusCode = response.map(_.code) + + assertZIO(body <&> statusCode)( + equalTo( + Right(PaginatedResponse(Seq(checkpointV2DTO2), Pagination(10, 0, hasMore = false), uuid)), + StatusCode.Ok + ) + ) + }, + test("Returns expected 400 when 'from' is not a valid date-time") { + val request = basicRequest + .get(uri"https://test.com/api/v2/partitionings/1/checkpoints?limit=10&offset=0&from=2026-06-01") + .response(asJson[PaginatedResponse[CheckpointV2DTO]]) + + val response = request + .send(backendStub) + + val statusCode = response.map(_.code) + + assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) + }, + test("Returns expected 400 when 'from' is not before 'to'") { + val request = basicRequest + .get( + uri"https://test.com/api/v2/partitionings/1/checkpoints?limit=10&offset=0&from=2026-08-01T00:00:00Z&to=2026-08-01T00:00:00Z" + ) + .response(asJson[PaginatedResponse[CheckpointV2DTO]]) + + val response = request + .send(backendStub) + + val statusCode = response.map(_.code) + + assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) + }, test("Returns an expected PaginatedResponse[CheckpointV2DTO] with more data available") { val request = basicRequest .get(uri"https://test.com/api/v2/partitionings/1/checkpoints?limit=10&offset=0") diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryUnitTests.scala index af5308870..ef54fb94e 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryUnitTests.scala @@ -62,11 +62,11 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { when(getCheckpointMockV2.apply(GetPartitioningCheckpointV2Args(partitioningId, checkpointV2DTO3.id))) .thenReturn(ZIO.fail(new Exception("boom!"))) - when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(0L, 10, 0L, None, None, None))) + when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(0L, 10, 0L, None, None, None, None, None))) .thenReturn(ZIO.right(Seq(Row(FunctionStatus(11, "ok"), Some(checkpointItemPaginatedFromDB))))) - when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(1L, 10, 0L, None, None, None))) + when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(1L, 10, 0L, None, None, None, None, None))) .thenReturn(ZIO.right(Seq(Row(FunctionStatus(11, "ok"), None)))) - when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(3L, 10, 0L, None, None, None))) + when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(3L, 10, 0L, None, None, None, None, None))) .thenReturn(ZIO.left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) when(getCheckpointPropertiesMock.apply(GetCheckpointPropertiesArgs(checkpointV2DTO1.id))) @@ -137,14 +137,14 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { suite("GetPartitioningCheckpointsSuite")( test("Returns expected Seq without properties") { for { - result <- CheckpointRepository.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, includeProperties = false) + result <- CheckpointRepository.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, None, None, includeProperties = false) } yield assertTrue( result.isInstanceOf[ResultHasMore[CheckpointV2DTO]] && result.data == Seq(checkpointV2DTO2) ) }, test("Returns expected Seq with properties") { for { - result <- CheckpointRepository.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, includeProperties = true) + result <- CheckpointRepository.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, None, None, includeProperties = true) } yield assertTrue( result.isInstanceOf[ResultHasMore[CheckpointV2DTO]] && result.data == Seq( checkpointV2DTO2.copy(properties = Some(Map("propName1" -> "propValue1"))) @@ -153,13 +153,13 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { }, test("Returns expected Seq.empty") { for { - result <- CheckpointRepository.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, includeProperties = false) + result <- CheckpointRepository.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, None, None, includeProperties = false) } yield assertTrue( result.isInstanceOf[ResultNoMore[CheckpointV2DTO]] && result.data == Seq.empty[CheckpointV2DTO] ) }, test("Returns expected NotFoundDatabaseError") { - assertZIO(CheckpointRepository.getPartitioningCheckpoints(3L, 10, 0L, None, None, None, includeProperties = false).exit)( + assertZIO(CheckpointRepository.getPartitioningCheckpoints(3L, 10, 0L, None, None, None, None, None, includeProperties = false).exit)( failsWithA[NotFoundDatabaseError] ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryUnitTests.scala index ff30f4b86..12c594385 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryUnitTests.scala @@ -38,7 +38,7 @@ object FlowRepositoryUnitTests extends ZIOSpecDefault with TestData { private val getFlowCheckpointsV2Mock = mock(classOf[GetFlowCheckpoints]) private val getCheckpointPropertiesMock: GetCheckpointProperties = mock(classOf[GetCheckpointProperties]) - when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(1, 1, 1L, None, None))) + when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(1, 1, 1L, None, None, None, None, None))) .thenReturn( ZIO.right( Seq( @@ -46,9 +46,9 @@ object FlowRepositoryUnitTests extends ZIOSpecDefault with TestData { ) ) ) - when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(2, 1, 1L, None, None))) + when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(2, 1, 1L, None, None, None, None, None))) .thenReturn(ZIO.right(Seq(Row(FunctionStatus(11, "success"), None)))) - when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(3, 1, 1L, None, None))) + when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(3, 1, 1L, None, None, None, None, None))) .thenReturn(ZIO.fail(DataNotFoundException(FunctionStatus(42, "Flow not found")))) when( @@ -67,12 +67,12 @@ object FlowRepositoryUnitTests extends ZIOSpecDefault with TestData { suite("GetFlowCheckpointsV2Suite")( test("Returns expected Right with CheckpointV2DTO without properties") { for { - result <- FlowRepository.getFlowCheckpoints(1, 1, 1L, None, None, includeProperties = false) + result <- FlowRepository.getFlowCheckpoints(1, 1, 1L, None, None, None, None, None, includeProperties = false) } yield assertTrue(result == ResultHasMore(Seq(checkpointWithPartitioningDTO1))) }, test("Returns expected Right with CheckpointV2DTO with properties") { for { - result <- FlowRepository.getFlowCheckpoints(1, 1, 1L, None, None, includeProperties = true) + result <- FlowRepository.getFlowCheckpoints(1, 1, 1L, None, None, None, None, None, includeProperties = true) } yield assertTrue( result == ResultHasMore( Seq(checkpointWithPartitioningDTO1.copy(properties = Some(Map("propName1" -> "propValue1")))) @@ -81,11 +81,11 @@ object FlowRepositoryUnitTests extends ZIOSpecDefault with TestData { }, test("Returns expected Right with CheckpointV2DTO") { for { - result <- FlowRepository.getFlowCheckpoints(2, 1, 1L, None, None, includeProperties = false) + result <- FlowRepository.getFlowCheckpoints(2, 1, 1L, None, None, None, None, None, includeProperties = false) } yield assertTrue(result == ResultNoMore(Seq.empty[CheckpointWithPartitioningDTO])) }, test("Returns expected DatabaseError") { - assertZIO(FlowRepository.getFlowCheckpoints(3, 1, 1L, None, None, includeProperties = false).exit)( + assertZIO(FlowRepository.getFlowCheckpoints(3, 1, 1L, None, None, None, None, None, includeProperties = false).exit)( failsWithA[NotFoundDatabaseError] ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceUnitTests.scala index e2c5406dc..eaba19979 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceUnitTests.scala @@ -45,9 +45,9 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { when(checkpointRepositoryMock.getCheckpoint(partitioningId, checkpointV2DTO2.id, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundDatabaseError("not found"))) - when(checkpointRepositoryMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, includeProperties = false)) + when(checkpointRepositoryMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultHasMore(Seq(checkpointV2DTO1)))) - when(checkpointRepositoryMock.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, includeProperties = false)) + when(checkpointRepositoryMock.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundDatabaseError("Partitioning not found"))) private val checkpointRepositoryMockLayer = ZLayer.succeed(checkpointRepositoryMock) @@ -92,13 +92,13 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { suite("GetPartitioningCheckpointsSuite")( test("Returns expected Right with Seq[CheckpointDTO]") { for { - result <- CheckpointService.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, includeProperties = false) + result <- CheckpointService.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, None, None, includeProperties = false) } yield assertTrue { result == ResultHasMore(Seq(checkpointV2DTO1)) } }, test("Returns expected NotFoundServiceError") { - assertZIO(CheckpointService.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, includeProperties = false).exit)( + assertZIO(CheckpointService.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, None, None, includeProperties = false).exit)( failsWithA[NotFoundServiceError] ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/service/FlowServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/service/FlowServiceUnitTests.scala index bcd867305..51233afbe 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/service/FlowServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/service/FlowServiceUnitTests.scala @@ -29,9 +29,9 @@ import zio.test._ object FlowServiceUnitTests extends ZIOSpecDefault with TestData { private val flowRepositoryMock = mock(classOf[FlowRepository]) - when(flowRepositoryMock.getFlowCheckpoints(1L, 1, 1L, None, None, includeProperties = false)) + when(flowRepositoryMock.getFlowCheckpoints(1L, 1, 1L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultHasMore(Seq(checkpointWithPartitioningDTO1)))) - when(flowRepositoryMock.getFlowCheckpoints(2L, 1, 1L, None, None, includeProperties = false)) + when(flowRepositoryMock.getFlowCheckpoints(2L, 1, 1L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundDatabaseError("Flow not found"))) private val flowRepositoryMockLayer = ZLayer.succeed(flowRepositoryMock) @@ -42,13 +42,13 @@ object FlowServiceUnitTests extends ZIOSpecDefault with TestData { suite("GetFlowCheckpointsV2Suite")( test("Returns expected PaginatedResult[CheckpointV2DTO]") { for { - result <- FlowService.getFlowCheckpoints(1L, 1, 1L, None, None, includeProperties = false) + result <- FlowService.getFlowCheckpoints(1L, 1, 1L, None, None, None, None, None, includeProperties = false) } yield assertTrue { result == ResultHasMore(Seq(checkpointWithPartitioningDTO1)) } }, test("Returns expected ServiceError") { - assertZIO(FlowService.getFlowCheckpoints(2L, 1, 1L, None, None, includeProperties = false).exit)( + assertZIO(FlowService.getFlowCheckpoints(2L, 1, 1L, None, None, None, None, None, includeProperties = false).exit)( failsWithA[NotFoundServiceError] ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDBUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDBUnitTests.scala index 87d2b75c6..1be1b3625 100644 --- a/server/src/test/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDBUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDBUnitTests.scala @@ -24,7 +24,7 @@ import java.util.UUID class CheckpointItemWithPartitioningFromDBUnitTests extends AnyFunSuiteLike with TestData { - test("groupAndConvertItemsToCheckpointWithPartitioningDTOs should group by idCheckpoint and sort by checkpointStartTime in descending order") { + test("groupAndConvertItemsToCheckpointWithPartitioningDTOs should group by idCheckpoint and keep the order of the DB rows") { val checkpointItem1 = CheckpointItemWithPartitioningFromDB( idCheckpoint = UUID.randomUUID(), checkpointName = "checkpoint1", @@ -57,14 +57,20 @@ class CheckpointItemWithPartitioningFromDBUnitTests extends AnyFunSuiteLike with hasMore = true ) - val checkpointItems = Seq(checkpointItem1, checkpointItem2) + // a second measure of the first checkpoint, i.e. another DB row of the same checkpoint + val checkpointItem1OtherMeasure = checkpointItem1.copy(measureName = "measure3", measuredColumns = Seq("col3")) - val result = CheckpointItemWithPartitioningFromDB.groupAndConvertItemsToCheckpointWithPartitioningDTOs(checkpointItems) + // earliest first, as returned by the DB for latest-first = false + val earliestFirst = CheckpointItemWithPartitioningFromDB + .groupAndConvertItemsToCheckpointWithPartitioningDTOs(Seq(checkpointItem1, checkpointItem1OtherMeasure, checkpointItem2)) + assert(earliestFirst.isRight) + assert(earliestFirst.toOption.get.map(_.id) == Seq(checkpointItem1.idCheckpoint, checkpointItem2.idCheckpoint)) + assert(earliestFirst.toOption.get.head.measurements.size == 2) - assert(result.isRight) - val checkpoints = result.toOption.get - - assert(checkpoints.size == 2) - assert(checkpoints.head.processStartTime.isAfter(checkpoints(1).processStartTime)) + // latest first, as returned by the DB by default + val latestFirst = CheckpointItemWithPartitioningFromDB + .groupAndConvertItemsToCheckpointWithPartitioningDTOs(Seq(checkpointItem2, checkpointItem1, checkpointItem1OtherMeasure)) + assert(latestFirst.isRight) + assert(latestFirst.toOption.get.map(_.id) == Seq(checkpointItem2.idCheckpoint, checkpointItem1.idCheckpoint)) } } From a26a3d0864b2e3d860700c87d117f4a91317b01d Mon Sep 17 00:00:00 2001 From: lsulak Date: Fri, 25 Sep 2026 23:17:56 +0200 Subject: [PATCH 07/21] feat: removing unnecessary files --- .planning/PROJECT.md | 71 ----------------------- .planning/codebase/ARCHITECTURE.md | 28 --------- .planning/codebase/CONCERNS.md | 26 --------- .planning/codebase/CONVENTIONS.md | 30 ---------- .planning/codebase/INTEGRATIONS.md | 27 --------- .planning/codebase/STACK.md | 39 ------------- .planning/codebase/STRUCTURE.md | 53 ----------------- .planning/codebase/TESTING.md | 30 ---------- .planning/config.json | 93 ------------------------------ 9 files changed, 397 deletions(-) delete mode 100644 .planning/PROJECT.md delete mode 100644 .planning/codebase/ARCHITECTURE.md delete mode 100644 .planning/codebase/CONCERNS.md delete mode 100644 .planning/codebase/CONVENTIONS.md delete mode 100644 .planning/codebase/INTEGRATIONS.md delete mode 100644 .planning/codebase/STACK.md delete mode 100644 .planning/codebase/STRUCTURE.md delete mode 100644 .planning/codebase/TESTING.md delete mode 100644 .planning/config.json diff --git a/.planning/PROJECT.md b/.planning/PROJECT.md deleted file mode 100644 index 9790da28e..000000000 --- a/.planning/PROJECT.md +++ /dev/null @@ -1,71 +0,0 @@ -# Project Context - -**Core Value:** -Atum must answer audit questions efficiently: in one filtered, paginated query, return the last *N* months of -measurements for a dataset *and its lineage*, with each measurement tagged with its dataset. Checkpoint filters -(name, multi-value properties such as `executionID IN (...)`, time window) combine server-side instead of being -applied one after another by the client. - -**What This Is:** -Enhancements to `database`, `atum-server` and `atum-reader` that implement the consumption pattern in -`adrs/02-consumption-pattern-for-audits.md`: server-side time windows, a unified reader filter API, lineage-complete -audit scope, partial-key parent lookup, and recording explicit lineage edges. Milestone v1.1 delivers the -first two; the rest is deferred. - -**Current milestone:** v1.1 Audit Consumption (ADR 002). See `.planning/ROADMAP.md` and `.planning/REQUIREMENTS.md`. - -**Constraints:** -- Must stay backward compatible for API and reader consumers. New query params and DB function params are optional - and trailing; deprecate before removing. -- A DB migration must not break a server that is still on the previous release (new function params get defaults). -- No re-architecture of checkpoint storage in this milestone. Table partitioning and normalized keys are deferred. - -## Requirements - -### Validated -- ✓ Read checkpoints by exact name (existing) -- ✓ Server-side pagination (existing) -- ✓ Server filters checkpoints by multi-value properties, e.g. `executionID IN (a,b,c)` — v1.0, CHK-01/02 -- ✓ Pagination is accurate over the combined name + multi-property filter — v1.0, CHK-03 - -### Active -- [ ] Server-side `from`/`to` time window on flow and partitioning checkpoint queries, backed by an index (Phase 1) -- [ ] Reader: one `getCheckpointsPage` with a combined name + multi-value properties + time-window filter, plus a - drain-all-pages helper (Phase 2; carries over v1.0 CHK-04/05) - -### Out of Scope -- Major architectural changes to checkpoint storage. -- Lineage-complete audit scope, partial-key partitioning search, recording lineage edges (deferred 2026-09-25). -- Access control for audit consumers. -- Changes to the agent's checkpoint write path. - -## Key Decisions - -| Decision | Rationale | Outcome | -|----------|-----------|---------| -| Update both server and reader | Server only supported `Map[String, String]`, so backend IN-list support had to come before the reader could expose it. | ✓ Server done (v1.0); reader in v1.1 Phase 2 | -| Composite B-tree `(fk_partitioning, process_start_time)` before BRIN | Both endpoints scan checkpoints per partitioning. `process_start_time` is client-supplied, so the physical ordering BRIN relies on is not guaranteed. | ✓ Adopted; BRIN deferred | -| Main flow ≠ lineage chain | Verified in SQL: `main-flow(X) = {X} ∪ descendants(X)`. ADR §3 step 2 misses the upstream unless X is a root. | Documented as a known limitation; fix deferred | -| Time window on `process_start_time` | Business time is what audits ask about. | ✓ Decided 2026-09-25 | -| Amend unreleased `V0.8.0.1`/`V0.8.0.2` in place | One function version per release. | ✓ Decided 2026-09-25 | -| `CREATE INDEX CONCURRENTLY`, one index per migration | No write lock on `checkpoints` while the index builds. | ✓ Decided 2026-09-25 | - -## Evolution - -This document evolves at phase transitions and milestone boundaries. - -**After each phase transition** (via `/gsd-transition`): -1. Requirements invalidated? → Move to Out of Scope with reason -2. Requirements validated? → Move to Validated with phase reference -3. New requirements emerged? → Add to Active -4. Decisions to log? → Add to Key Decisions -5. "What This Is" still accurate? → Update if drifted - -**After each milestone** (via `/gsd-complete-milestone`): -1. Full review of all sections -2. Core Value check — still the right priority? -3. Audit Out of Scope — reasons still valid? -4. Update Context with current state - ---- -*Last updated: 2026-09-25 — milestone v1.1 (ADR 002) started* diff --git a/.planning/codebase/ARCHITECTURE.md b/.planning/codebase/ARCHITECTURE.md deleted file mode 100644 index ee6921951..000000000 --- a/.planning/codebase/ARCHITECTURE.md +++ /dev/null @@ -1,28 +0,0 @@ -# Architecture - -**Analysis Date:** 2026-09-24 - -## System Design - -**Core Concept:** -The Atum Service collects and stores measured data from Spark applications and allows retrieval. - -**Key Components:** -- `agent`: Library plugged into Spark applications to measure data and send it to the server. -- `server`: ZIO/HTTP4S-based API service that receives, stores, and serves measured data. -- `model`: Shared data model between agent, server, and reader. -- `database`: Relational database schema (PostgreSQL) and Doobie access patterns. -- `reader`: Client library for reading measured data from the server. - -## Data Flow -1. Spark application with `agent` measures data. -2. `agent` serializes data using `model` and sends HTTP request to `server`. -3. `server` validates and persists data into `database` using ZIO and Doobie. -4. Client application uses `reader` (and `model`) to fetch data from `server` via HTTP. - -## Key Abstractions -- **Effect System:** ZIO is heavily used in `server` for dependency injection and effect management. -- **API Definition:** Tapir is used to define HTTP endpoints declaratively, providing swagger UI and decoupling from HTTP4S. -- **Cross-building:** SBT `projectMatrix` is used for supporting different Scala/Spark combinations across modules. - - diff --git a/.planning/codebase/CONCERNS.md b/.planning/codebase/CONCERNS.md deleted file mode 100644 index 21cfbefac..000000000 --- a/.planning/codebase/CONCERNS.md +++ /dev/null @@ -1,26 +0,0 @@ -# Codebase Concerns - -**Analysis Date:** 2026-09-24 - -## Tech Debt - -**Java Version Requirements:** -- Issue: `build.sbt` contains a check limiting project loading if Java is older than the recommended version (warning on Java version affecting `atum-server` and `atum-database`). -- Files: `build.sbt` -- Fix approach: Ensure build environment always meets the recommended Java version (Java 11 or 17). - -## Security Considerations - -**AWS Credentials:** -- `server` integrates with AWS Secrets Manager and STS. Ensure local dev doesn't check in AWS keys. Use IAM roles where possible. - -## Fragile Areas - -**Cross-Building:** -- Issue: Heavy use of `projectMatrix` to build for multiple Spark/Scala versions. This can make the build complex and brittle to dependency upgrades (e.g., Spark 3.5.5 update might require updating other test libraries). - -## Missing Critical Features - -- No immediate missing features identified during map generation. Look at issues list for more context. - - diff --git a/.planning/codebase/CONVENTIONS.md b/.planning/codebase/CONVENTIONS.md deleted file mode 100644 index 748af6be1..000000000 --- a/.planning/codebase/CONVENTIONS.md +++ /dev/null @@ -1,30 +0,0 @@ -# Coding Conventions - -**Analysis Date:** 2026-09-24 - -## Code Style - -**Formatting:** -- `scalafmt` is used (indicated by `.scalafmt.conf`) -- Scala idiomatic style - -## Error Handling - -**Patterns:** -- In `server`: ZIO effects (`ZIO[R, E, A]`) are used to handle domain errors (`E`) gracefully. -- Typed errors mapped to HTTP responses using Tapir error mappings. -- In `agent` / `model`: standard Scala `Try`, `Either`, and exceptions (e.g., `AtumAgentException`). - -## Logging - -**Framework:** -- `zio-logging` and `logback` used heavily in `server` -- `logback` for SLF4J in `agent` - -## Function Design - -- Functional Programming principles heavily used -- Case classes for immutable data structures -- Type classes for JSON serialization (`circe` codecs) - - diff --git a/.planning/codebase/INTEGRATIONS.md b/.planning/codebase/INTEGRATIONS.md deleted file mode 100644 index 24cc1b688..000000000 --- a/.planning/codebase/INTEGRATIONS.md +++ /dev/null @@ -1,27 +0,0 @@ -# Integrations - -**Analysis Date:** 2026-09-24 - -## External APIs - -**AWS Services:** -- AWS Secrets Manager (`software.amazon.awssdk:secretsmanager`) -- AWS STS (`software.amazon.awssdk:sts`) - -**Data Processing:** -- Apache Spark (via `agent` plugin) - -## Databases - -**PostgreSQL:** -- Connected via `doobie` / `fa-db` in the `server` module -- Migrations managed via Flyway - -## Metrics & Observability - -**Prometheus:** -- `zio-metrics-connectors-prometheus` -- `http4s-prometheus-metrics` -- `tapir-prometheus-metrics` - - diff --git a/.planning/codebase/STACK.md b/.planning/codebase/STACK.md deleted file mode 100644 index ab8fad964..000000000 --- a/.planning/codebase/STACK.md +++ /dev/null @@ -1,39 +0,0 @@ -# Tech Stack - -**Analysis Date:** 2026-09-24 - -## Core Technologies - -**Languages:** -- Scala (2.13 primary) -- Java (Required/Recommended bounds via build.sbt) - -**Runtimes & Frameworks:** -- Apache Spark (3.5.5) - For the `agent` module processing data -- ZIO (2.0.19) - Core effect system and DI for `server` -- HTTP4S (0.23) & Tapir (1.9) - HTTP server and API definitions -- STTP (3.5/3.9) - HTTP client - -## Infrastructure - -**Databases:** -- PostgreSQL (Flyway for migrations via `sbt-flyway`) -- Doobie / `fa-db` (0.7.0) for DB access - -**Cloud & Deployment:** -- AWS SDK (SecretsManager, STS) - -## Key Dependencies - -**Libraries:** -- `circe` for JSON serialization -- `scalatest` & `mockito-scala` & `zio-test` & `specs2` for testing -- `logback` & `zio-logging` for logging - -## Configuration - -**Tools:** -- SBT (build tool, `projectMatrix` used for cross-building) -- TypeSafe Config & ZIO Config - - diff --git a/.planning/codebase/STRUCTURE.md b/.planning/codebase/STRUCTURE.md deleted file mode 100644 index 2a1df4837..000000000 --- a/.planning/codebase/STRUCTURE.md +++ /dev/null @@ -1,53 +0,0 @@ -# Codebase Structure - -**Analysis Date:** 2026-09-24 - -## Directory Layout - -``` -/ -├── adrs/ # Architecture Decision Records -├── agent/ # Spark plugin for measuring and sending data -├── api-tests/ # Integration and E2E tests for the API -├── database/ # Database migrations and Doobie fragments -├── model/ # Shared domain models -├── project/ # SBT build configuration and dependencies -├── reader/ # Client library for reading data from server -└── server/ # The ZIO HTTP4s Tapir REST API server -``` - -## Directory Purposes - -**`agent/`:** -- Purpose: Spark library to capture metrics -- Contains: Scala code - -**`server/`:** -- Purpose: Backend service -- Contains: ZIO services, Tapir endpoints, Doobie queries - -**`model/`:** -- Purpose: Shared DTOs -- Contains: Case classes and Circe codecs - -**`database/`:** -- Purpose: DB definition -- Contains: Flyway SQL migrations - -## Entry Points - -**Configuration:** -- `build.sbt`: Multi-module SBT definition -- `project/Dependencies.scala`: All library versions - -## Where to Add New Code - -**New API Endpoint:** -- Define in `server/src/.../endpoints` using Tapir -- Implement logic in `server/src/.../services` -- Add shared DTOs to `model/` - -**New Spark metric:** -- Add to `agent/` - - diff --git a/.planning/codebase/TESTING.md b/.planning/codebase/TESTING.md deleted file mode 100644 index 8af6576f1..000000000 --- a/.planning/codebase/TESTING.md +++ /dev/null @@ -1,30 +0,0 @@ -# Testing Patterns - -**Analysis Date:** 2026-09-24 - -## Test Framework - -**Runners:** -- `scalatest` (3.2) - primary for non-ZIO modules (`agent`, `model`, `reader`) -- `zio-test` - primary for the `server` module -- `specs2` - also present in `model` module dependencies - -**Mocking:** -- `mockito-scala` for ScalaTest mocking - -## Test Structure - -- Standard SBT structure (`src/test/scala`) -- `api-tests/` directory contains HTTP API integration tests (potentially using `balta` or HTTP clients) - -## Run Commands -```bash -sbt test -sbt "project server" test -``` - -## Mocking -- ZIO environment (`ZLayer`) is used for dependency injection and mocking in `server` tests. -- `mockito-scala` used in `agent` for mocking Spark or HTTP clients. - - diff --git a/.planning/config.json b/.planning/config.json deleted file mode 100644 index f62bd6dc7..000000000 --- a/.planning/config.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "model_profile": "inherit", - "commit_docs": true, - "parallelization": true, - "search_gitignored": false, - "brave_search": false, - "firecrawl": false, - "exa_search": false, - "tavily_search": false, - "ref_search": false, - "perplexity": false, - "jina": false, - "git": { - "branching_strategy": "none", - "create_tag": true, - "phase_branch_template": "gsd/phase-{phase}-{slug}", - "milestone_branch_template": "gsd/{milestone}-{slug}", - "quick_branch_template": null - }, - "workflow": { - "research": false, - "plan_check": true, - "verifier": true, - "nyquist_validation": true, - "auto_advance": false, - "node_repair": true, - "node_repair_budget": 2, - "ui_phase": true, - "ui_safety_gate": true, - "ai_integration_phase": true, - "api_coverage_gate": true, - "human_verify_mode": "end-of-phase", - "context_guard_mode": "warn", - "text_mode": false, - "research_before_questions": false, - "discuss_mode": "discuss", - "skip_discuss": false, - "code_review": true, - "code_review_depth": "standard", - "code_review_command": null, - "pattern_mapper": true, - "plan_bounce": false, - "plan_bounce_script": null, - "plan_bounce_passes": 2, - "auto_prune_state": false, - "post_planning_gaps": true, - "security_enforcement": true, - "security_asvs_level": 1, - "security_block_on": "high", - "compact_content": false - }, - "ship": { - "pr_body_sections": [ - { - "heading": "User Stories & Acceptance Criteria", - "enabled": true, - "source": "REQUIREMENTS.md ## User Stories || REQUIREMENTS.md ## Acceptance Criteria", - "fallback": "- Acceptance criteria are covered by the linked requirements and verification evidence." - }, - { - "heading": "Risks & Dependencies", - "enabled": true, - "source": "PLAN.md ## Risks || PLAN.md ## Dependencies", - "fallback": "- No known high-risk rollout dependencies." - }, - { - "heading": "Success Metrics & Release Criteria", - "enabled": true, - "source": "REQUIREMENTS.md ## Definition of Done || VERIFICATION.md ## Release Criteria", - "fallback": "- Release when automated verification and required manual checks pass." - }, - { - "heading": "Stakeholder Review & Approval", - "enabled": true, - "template": "- Product owner approval pending for {phase_name}." - } - ] - }, - "hooks": { - "context_warnings": true - }, - "project_code": null, - "phase_naming": "sequential", - "agent_skills": {}, - "claude_md_path": "./.hermes/HERMES.md", - "plan_review": { - "source_grounding": true, - "source_grounding_authority": "grep" - }, - "resolve_model_ids": "omit", - "mode": "yolo", - "granularity": "standard" -} From 478a4212614633ccd48bacf076fcfee11d7f0f0d Mon Sep 17 00:00:00 2001 From: lsulak Date: Fri, 25 Sep 2026 23:22:17 +0200 Subject: [PATCH 08/21] updating readme --- README.md | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/README.md b/README.md index c23482cea..6a078e4f2 100644 --- a/README.md +++ b/README.md @@ -232,39 +232,6 @@ represents all currently supported measurement types (aka measures): [//]: # (| controlType.absAggregatedTruncTotal | Calculates SUM(TRUNC(ABS())) of the specified column |) -### Reading checkpoints - -The _Atum Reader_ reads checkpoints of a partitioning (`PartitioningReader`) or of the whole flow a partitioning is the -main partitioning of (`FlowReader`). The checkpoints to read are described by a `CheckpointFilter`; its parts are all -optional and combined together: - -| Filter part | Meaning | -|---------------|:-----------------------------------------------------------------------------------------------| -| `name` | checkpoints of this name | -| `properties` | for every property name given, the checkpoint property has one of the values (`IN (...)`) | -| `from`, `to` | process start time window: `from` inclusive, `to` exclusive | -| `latestFirst` | order by process start time, latest first (default) or earliest first | - -For example, all measurements of a flow from the last two months, of two specific executions: - -```scala -import za.co.absa.atum.reader.FlowReader -import za.co.absa.atum.reader.requests.CheckpointFilter - -val filter = CheckpointFilter( - properties = Map("executionID" -> Set("a", "b")), - from = Some(ZonedDateTime.now().minusMonths(2)) -) -val reader = FlowReader(mainFlowPartitioning) - -reader.getCheckpointsPage(pageSize = 100, offset = 0, filter = filter) // one page -reader.getAllCheckpoints(filter) // all pages, held in memory -``` - -Each flow checkpoint carries the partitioning it belongs to. The flow of a partitioning covers the partitioning and the -partitionings derived from it (its descendants), not its ancestors. - - ## How to generate Code coverage report - Use java version 11. From 23c51bb17564d24c5a06db73ce4cb350a097a26e Mon Sep 17 00:00:00 2001 From: lsulak Date: Fri, 9 Oct 2026 17:11:32 +0200 Subject: [PATCH 09/21] refactor: move audit scope (ADR 02) out of the #478 reader capability work Keep this change focused on #478 (combined name + multi-value properties filter with accurate server-side pagination). The audit-oriented additions move to a follow-up PR: - process start time window (from / to) and latest-first on the flows endpoint, through all layers - CheckpointFilter.from / to / latestFirst, getAllCheckpoints and Reader.queryAllPages - flow checkpoints DB-order preservation in CheckpointItemWithPartitioningFromDB - runs.checkpoints (fk_partitioning, process_start_time) index - ADR 02 - consumption pattern for audits (cherry picked from commit 4ce3cae03710ea1ffe82e3da1c43e446d2f6f504) --- .gitignore | 1 - README.md | 2 +- adrs/02-consumption-pattern-for-audits.md | 324 ------------------ .../flows/V0.8.0.1__get_flow_checkpoints.sql | 20 +- ...V0.8.0.2__get_partitioning_checkpoints.sql | 20 +- ...ckpoints_partitioning_start_time_index.ddl | 23 -- ...checkpoint_properties_checkpoint_index.ddl | 23 -- .../GetFlowCheckpointsIntegrationTests.scala | 144 +------- ...titioningCheckpointsIntegrationTests.scala | 116 +------ .../za/co/absa/atum/reader/FlowReader.scala | 30 +- .../absa/atum/reader/PartitioningReader.scala | 31 +- .../za/co/absa/atum/reader/core/Reader.scala | 22 -- .../reader/requests/CheckpointFilter.scala | 32 +- .../reader/requests/QueryParamNames.scala | 3 - .../atum/reader/FlowReaderUnitTests.scala | 63 +--- .../reader/PartitioningReaderUnitTests.scala | 30 +- .../requests/CheckpointFilterUnitTests.scala | 16 +- .../flows/functions/GetFlowCheckpoints.scala | 12 +- .../GetPartitioningCheckpoints.scala | 10 +- .../v2/controller/CheckpointController.scala | 3 - .../controller/CheckpointControllerImpl.scala | 5 - .../api/v2/controller/FlowController.scala | 5 - .../v2/controller/FlowControllerImpl.scala | 17 +- .../atum/server/api/v2/http/Endpoints.scala | 37 +- .../v2/repository/CheckpointRepository.scala | 3 - .../repository/CheckpointRepositoryImpl.scala | 14 +- .../api/v2/repository/FlowRepository.scala | 5 - .../v2/repository/FlowRepositoryImpl.scala | 9 +- .../api/v2/service/CheckpointService.scala | 3 - .../v2/service/CheckpointServiceImpl.scala | 5 - .../server/api/v2/service/FlowService.scala | 5 - .../api/v2/service/FlowServiceImpl.scala | 17 +- ...CheckpointItemWithPartitioningFromDB.scala | 6 +- .../GetFlowCheckpointsIntegrationTests.scala | 29 +- ...titioningCheckpointsIntegrationTests.scala | 14 +- .../CheckpointControllerUnitTests.scala | 6 +- .../controller/FlowControllerUnitTests.scala | 12 +- .../GetFlowCheckpointsEndpointUnitTests.scala | 88 +---- ...itioningCheckpointsEndpointUnitTests.scala | 72 +--- .../CheckpointRepositoryUnitTests.scala | 14 +- .../repository/FlowRepositoryUnitTests.scala | 14 +- .../service/CheckpointServiceUnitTests.scala | 8 +- .../api/v2/service/FlowServiceUnitTests.scala | 8 +- ...tItemWithPartitioningFromDBUnitTests.scala | 22 +- 44 files changed, 113 insertions(+), 1230 deletions(-) delete mode 100644 adrs/02-consumption-pattern-for-audits.md delete mode 100644 database/src/main/postgres/runs/V0.8.0.3__checkpoints_partitioning_start_time_index.ddl delete mode 100644 database/src/main/postgres/runs/V0.8.0.4__checkpoint_properties_checkpoint_index.ddl diff --git a/.gitignore b/.gitignore index 4e2a872b7..a864bff58 100644 --- a/.gitignore +++ b/.gitignore @@ -92,4 +92,3 @@ utils/resources/*.conf /server/selfsigned.crt /server/selfsigned.p12 /.bloop/ -.planning/ diff --git a/README.md b/README.md index c482ed461..8e9880fb2 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,6 @@ - [Usage](#usage) - [Atum Agent routines](#atum-agent-routines) - [Control measurement types](#control-measurement-types) - - [Reading checkpoints](#reading-checkpoints) - [How to generate Code coverage report](#how-to-generate-code-coverage-report) - [How to Run in IntelliJ](#how-to-run-in-intellij) - [How to Run Tests](#how-to-run-tests) @@ -373,6 +372,7 @@ represents all currently supported measurement types (aka measures): [//]: # (| controlType.absAggregatedTruncTotal | Calculates SUM(TRUNC(ABS())) of the specified column |) + ## How to generate Code coverage report - Use java version 11. diff --git a/adrs/02-consumption-pattern-for-audits.md b/adrs/02-consumption-pattern-for-audits.md deleted file mode 100644 index 5afdcf88c..000000000 --- a/adrs/02-consumption-pattern-for-audits.md +++ /dev/null @@ -1,324 +0,0 @@ -# ADR 02 — Consumption Pattern for Audit Queries (Time-Windowed Lineage & Measurements) - -| | | -|-------------------------|------------------------------------------------------------------------------------------------------------------| -| **Status** | Proposed | -| **Date** | 2026-09-20 | -| **Deciders** | Atum Service maintainers, platform architects, audit/consumer stakeholders | -| **Affected components** | `server/` + `database/` (optional time-filter enhancement only), `reader/` (consumption client — already exists) | -| **Implementation** | §4 (time window, index, `latest-first` on flows) and its reader support done (2026-09-25); §5 and §6 deferred | - ---- - -## 1. Context — what an auditor actually asks for - -A typical audit question is: - -> "For this dataset, show me the **last 2 months** of integrity measurements, and **how they relate to each -> other** — i.e. which measurement belongs to which dataset, and which dataset was produced from which upstream." - -That is two needs: - -1. A **time window** (e.g. the last 2 months). -2. The **relationships** between measurements — each measurement's dataset, and the lineage between datasets (what - produced what). - -This ADR describes how to satisfy both using the existing relational model and service API, and the one small -addition that makes the time window efficient. - ---- - -## 2. What the relational model already gives us - -Three facts about the current schema make this almost entirely a "use what's there" exercise: - -- **A flow is a partitioning plus everything derived from it.** Every partitioning gets its own *main flow* when it - is created (`flows._create_flow`), and linking a child to a parent (including across applications) adds the child - to **all flows of the parent** (`flows._add_to_parent_flows` → `flows.partitioning_to_flow`). So the main flow of a - partitioning = the partitioning **and its descendants** — the "set of related datasets" downstream of it. It does - **not** contain the partitioning's ancestors; only the main flow of a root covers the whole lineage chain. -- **One call returns every measurement in that chain, already tagged with its dataset.** - `flows.get_flow_checkpoints(flowId, ...)` returns all checkpoints across the whole flow, and **every returned row - carries the `id_partitioning` + `partitioning` JSON it belongs to** — plus `measure_name`, `measurement_value`, - `checkpoint_start_time`, `checkpoint_end_time`, author, etc. That single call delivers *both* the measurements *and* - their relationships (measurement→dataset; the shared flow ties datasets into one lineage graph). It is - exposed as `GET /api/v2/flows/{flowId}/checkpoints`. -- **The time data already exists.** `runs.checkpoints.process_start_time` (and `process_end_time`) are stored, and - `get_flow_checkpoints` already returns results **latest-first** (ordered by `process_start_time DESC`). - -A ready-made client also already exists: the **`reader`** module's `FlowReader`, which does exactly the -partitioning → flow → paged-checkpoints walk described below. - -``` - Main flow of "AQ dataset" = AQ dataset + everything derived from it - - AQ dataset ─────▶ UU domain ─────▶ UU feed - │ │ │ - checkpoints checkpoints checkpoints - (measures) (measures) (measures) - - GET /flows/{flowId}/checkpoints ── returns ALL of these rows, each tagged - with the partitioning it belongs to. - - The main flow of "UU feed", however, contains only UU feed (and whatever is derived from it). -``` - ---- - -## 3. The consumption pattern (recipe) - -Three steps, all on **today's** v2 API: - -1. **Resolve the starting dataset → partitioning id.** - `GET /api/v2/partitionings?partitioning=` → `{ id }`. *(Resolving that starting - point from a business key such as `catalog_path`)* -2. **Resolve its lineage chain → flow id.** - `GET /api/v2/partitionings/{id}/main-flow` → `{ id: }`. - This covers the dataset and its **descendants**. To include the **upstream** datasets too, start from the root: - `GET /api/v2/partitionings/{id}/ancestors` lists the primary partitionings of the flows the dataset belongs to; - take the root among them and use *its* main flow. (Known limitation: a partitioning re-parented with - `PATCH .../ancestors` joins the new parent's flows, but its already existing descendants do not.) -3. **Pull all measurements across the chain, newest first, paged.** - `GET /api/v2/flows/{flowId}/checkpoints?limit=100&offset=0&include-properties=true`. - Each row = one measure of one checkpoint, **tagged with its partitioning**. That is the audit dataset: - measurements + their dataset relationships across the whole lineage chain. - -### Worked example (HTTP) - -```http -# 1) dataset (partitioning JSON, base64-encoded) -> its id -GET /api/v2/partitionings?partitioning=eyJhdWxfZG9tYWluX3V1aWQiOiAiYTZ... -> { "id": 2000000010448528 } - -# 2) id -> its flow (lineage chain) -GET /api/v2/partitionings/2000000010448528/main-flow -> { "id": 3000000000123456 } - -# 3) flow -> every measurement in the chain, latest first, one page at a time -GET /api/v2/flows/3000000000123456/checkpoints?limit=100&offset=0&include-properties=true -``` - -Each page item carries `checkpointStartTime`, `measureName`/`measurementValue`, and the `partitioning` it belongs -to — so relationships come for free. - -### "Last 2 months" today (no server change needed) - -Because results are latest-first, page step 3 and **stop at the first row older than the cutoff**: - -``` -keep the page while checkpointStartTime >= now - 2 months -stop as soon as a row is older (all later rows are older too) -``` - -This works on the current API. Its only cost is over-fetching whole pages and re-implementing the cutoff in every -consumer. - -### Pure lineage topology (optional) - -If an audit wants the dataset graph *without* the measurements, use `GET /api/v2/flows/{flowId}/partitionings` -(all datasets in the chain) and/or `GET /api/v2/partitionings/{id}/ancestors` (a dataset's upstream roots). - ---- - -## 4. The one gap, and the proposed change - -**Gap:** there is **no server-side time filter** on any checkpoint endpoint today. `get_flow_checkpoints` filters -by `checkpoint-name` and `checkpoint-properties` only; "last 2 months" must be done client-side (above). - -**Proposed change — minimal and backward-compatible** (mirrors the *existing* optional filters, so nothing breaks). -*Implemented (2026-09-25), with the index choice described below:* - -- **Database:** add two optional parameters to `flows.get_flow_checkpoints` - (and, for the single-dataset case, `runs.get_partitioning_checkpoints`): - - ```sql - -- new optional params, both default NULL (= no filter, current behaviour) - i_from_time TIMESTAMP WITH TIME ZONE DEFAULT NULL, - i_to_time TIMESTAMP WITH TIME ZONE DEFAULT NULL - -- ...added to the WHERE clause: - AND (i_from_time IS NULL OR C.process_start_time >= i_from_time) - AND (i_to_time IS NULL OR C.process_start_time < i_to_time) - ``` - -- **API:** add optional `from` / `to` (ISO-8601) query params to `GET /api/v2/flows/{flowId}/checkpoints` - (and `GET /api/v2/partitionings/{id}/checkpoints`): - - ```http - GET /api/v2/flows/{flowId}/checkpoints?from=2026-06-01T00:00:00Z&to=2026-08-01T00:00:00Z&limit=100 - ``` - -- **Index:** add a supporting index — today the only index on `runs.checkpoints` is on `fk_partitioning`, so a large - flow's time-window scan has nothing to lean on. *Implemented as the composite `(fk_partitioning, process_start_time)` - (`checkpoints_idx2`, built `CONCURRENTLY`)*: both endpoints read checkpoints per partitioning, so each partitioning - of the flow is range-scanned by time. On 2M checkpoints (200k in the flow) a 2-month page of 100 took ~26 ms with - it and ~1.2 s without it. The checkpoint properties filter got a supporting - `runs.checkpoint_properties (fk_checkpoint, property_name)` index as well. - -With this, "last 2 months" is a single server-side query per page instead of client-side over-fetching, and the -cutoff logic lives in one place. - -*(Minor: the `/flows/{flowId}/checkpoints` endpoint now exposes `latest-first` too — the DB function has it and -defaults to `TRUE`, which is what audits want anyway.)* - -**Consumption via the reader:** `FlowReader` / `PartitioningReader` take a `CheckpointFilter` (name, multi-value -properties, `from` / `to`, `latestFirst`) in `getCheckpointsPage`, and `getAllCheckpoints` reads all the pages of a -filtered query. - ---- - -## 5. Partial parent lookup — whole-dataset / "ignore a key" - -A related consumer need (not audit-specific): **link to / load an upstream *parent* partitioning when you cannot -supply the full key.** Two triggers: you read the **whole** dataset without partition pruning (the upstream is -partitioned by `currency` and you want *all* currencies), or the key simply doesn't apply to you (the upstream -isn't partitioned by it at all). - -It splits into two directions, and **only one is actually hard**: - -- **Too many keys — "ignore one."** You computed `{catalog_path, info_date, currency}` but want to ignore `currency`. - You control your own query, so just **drop the key and do the normal exact lookup**. Nothing new is needed. -- **Too few keys — "all currencies."** You only know `{catalog_path, info_date}`; the stored parents are - `{…, currency: USD}`, `{…, currency: EUR}`, …, and you cannot enumerate the values. This is the one case that - needs more than an exact match. - -### Preferred: a *rollup* parent (no new Atum capability) - -The relational model already represents "less-specific parent → more-specific child": `mergeWithParent = true` -makes `child = parent ++ sub`, and AQ already relies on it (a job-level parent whose children merely *add* -`catalog_path` for example). So a producer that partitions by `currency` registers a **rollup node** and the -per-value partitions beneath it, using the existing Agent API: - -```scala -val rollup = AtumAgent.getOrCreateAtumContext( - AtumPartitions("catalog_path" -> x, "info_date" -> y)) // the "whole dataset" node -val usd = rollup.subPartitionContext(AtumPartitions("currency" -> "USD")) // -> {..., currency: USD} -``` - -Consumption then stays **all exact-match**: - -- **whole dataset** → exact-match the **rollup** `{catalog_path, info_date}` and link to it; every currency - partition is reachable transitively through the flow (`.../ancestors`, `.../flows/{flowId}/checkpoints`). -- **one currency** → exact-match `{..., currency: USD}`. -- **ignore currency** → drop it → exact-match the rollup. - -This is the most Atum-idiomatic answer: no new query capability, no schema change, and it reuses the parent-child -flow model, so lineage still connects the rollup to each partition. - -### Fallback: containment search (only when no rollup is guaranteed) - -The rollup only helps if the producer emits it. When a producer emits **only** per-value nodes and a consumer -reads the whole dataset, exact match cannot reach them. That single case is what a genuinely new **subset / -containment** lookup is for: - -- **DB:** - `runs.get_partitionings_by_keys(i_keys JSONB, i_limit, i_offset) … WHERE partitioning -> 'keysToValuesMap' @> i_keys` - — containment must target the `keysToValuesMap` sub-object, because the stored value is the envelope - `{ keys, version, keysToValuesMap }` (with an ordered `keys` array), **not** a flat map — backed by a - `jsonb_path_ops` **GIN index** on `(partitioning -> 'keysToValuesMap')` (none exists today — the store is - exact-match only). -- **API:** `GET /api/v2/partitionings/search?keys=&limit=&offset=` (mirrors the existing - base64-JSON convention of `GET /partitionings`). -- **0 / 1 / many results:** a partial key legitimately matches many parents — link to **all** of them (the flow - model is many-to-many, so N parents need no schema change), or, for the least-selective `catalog_path`-only - case, the most recent one. - -**Recommendation:** prefer the rollup convention; treat containment search as the fallback for heterogeneous or -uncooperative producers (and ad-hoc audits). - ---- - -## 6. Bigger bets — conceptual changes to the model or service - -Everything above works on today's schema. This section steps back and asks the question directly: *if we were -willing to change the relational model or the service, what would make lineage + audit consumption dramatically -easier?* These are **options to inform the roadmap, not commitments** — most are independent and can be adopted -incrementally. They are ordered cheapest-and-most-compatible first. - -Two facts (both verified in the schema) frame the whole list: - -- **A partitioning is an opaque, exact-match blob.** `runs.partitionings.partitioning` is a single `JSONB` under a - `UNIQUE` B-tree over the *whole* value; that value is an **envelope** — - `{ "keys": [...], "version": 1, "keysToValuesMap": { ... } }` — where `keys` is an *ordered array*. So identity - is the entire structure (key order included), and there is **no way to query by an individual dimension** or by a - key *subset* without a sequential scan. -- **Lineage is stored only as flow membership, not as edges.** `get_partitioning_ancestors` returns the *flow - roots* a partitioning sits under (it joins `partitioning_to_flow -> flows -> fk_primary_partitioning`). - The **immediate parent->child edge and the intermediate topology are not persisted** — only "belongs to a flow whose - root is R." The true lineage *graph* therefore cannot be reconstructed from the DB today. - -### Bet 1 — Make partition keys queryable (index, then optionally normalize) - -- **Idea (two sizes).** *Small & backward-compatible:* add a `jsonb_path_ops` **GIN index on - `(partitioning -> 'keysToValuesMap')`** plus a helper `get_partitionings_by_keys(...) WHERE - partitioning -> 'keysToValuesMap' @> i_keys`. *Larger:* maintain a normalized child table - `runs.partitioning_keys(fk_partitioning, key, value)` (a trigger keeps it in sync), or generated columns for the - hot linkage keys (`catalog_path`, `info_date`). -- **Unlocks.** Subset / containment lookup ("all currencies"), query-by-dimension ("every partitioning where - `currency = EUR`"), and partial-parent lookup (§5) become **indexed** SQL instead of scans. This is the single - change that turns **§5's hard case** (containment search) from "new subsystem" into "one index + one - function." -- **Cost.** The GIN index is nearly free and fully backward-compatible (no write-path change). The normalized - table costs a trigger, a backfill, and storage, and must be kept consistent with the JSONB. -- **Verdict.** **Do the GIN index first** — highest value-to-effort in the whole list. Promote to a normalized - table only if dimension queries become a first-class, high-volume access pattern. - -### Bet 2 — Make time a first-class checkpoint dimension - -- **Idea.** The infrastructure behind §4: add a **BRIN index on `runs.checkpoints (process_start_time)`** (ideal - for an append-only, time-ordered table and physically tiny), and consider native **range partitioning** of - `checkpoints` by time once volume warrants. Pair with the `from`/`to` params from §4. -- **Unlocks.** Cheap time-window audits **at scale** — the literal §1 question ("last 2 months") answered - server-side instead of by client over-fetch — and old partitions become trivially archivable. -- **Cost.** BRIN index: low and backward-compatible. Table partitioning: medium (migration), defer until needed. -- **Verdict.** **Do the BRIN index with §4.** It is the cheapest structural change that scales the core audit - query. -- **Outcome (2026-09-25).** §4 shipped with the composite B-tree `(fk_partitioning, process_start_time)` instead: - checkpoint queries are always scoped to partitionings, and `process_start_time` is supplied by the client (late or - backfilled reports), so the physical time ordering BRIN depends on is not guaranteed. BRIN is deferred until - `EXPLAIN` at production volume shows a need. - -### Bet 3 — Persist lineage as explicit edges (a real DAG) - -- **Idea.** Add `runs.partitioning_edges(fk_parent, fk_child, created_by, created_at)` capturing the **immediate** - parent->child relationship at link time. Keep `flows` / `partitioning_to_flow` as a denormalized reachability - cache - or, longer term, *derive* it from edges. -- **Unlocks.** The exact **edge list** and immediate parent/child (impossible today, per the framing note above), - arbitrary **N-hop traversal** via a recursive CTE, a true multi-parent DAG **with provenance** (who linked what, - when), and a real graph endpoint (`nodes + edges`) for audits that must *show the lineage*, not just the set. -- **Cost.** One new table plus dual-maintenance with flows (medium), or a larger refactor if flows become a pure - projection of edges (higher). -- **Verdict.** Recommend at minimum *recording* edges now (cheap, additive) even before building graph queries — the - provenance is impossible to reconstruct later. - -### Summary - -**Recommended sequence.** Ship the two low-cost, backward-compatible indexes first — **Bet 1's GIN** and **Bet 2's -BRIN** (no producer changes, and together they unlock most of the audit + partial-lookup value). Start **recording -edges** (Bet 3) additively so provenance isn't lost. *(Bet 2 was covered by the composite B-tree of §4; Bets 1 and 3 -are deferred.)* - ---- - -## 7. Alternatives Considered - -- **Client-side windowing only (status quo).** Page latest-first and stop at the cutoff. Zero server change; - acceptable as an interim. Downside: over-fetches whole pages near the boundary, and every consumer must - re-implement the cutoff. -- **Iterate per partitioning** (`get_partitioning_checkpoints` for each dataset). Rejected: the caller would first - have to enumerate the flow's datasets and then fan out N calls, re-implementing what `get_flow_checkpoints` - already does server-side in one call. -- **A new bespoke "audit export" endpoint.** Rejected for now: the flow-checkpoints endpoint plus a time filter - already returns exactly the needed shape; a dedicated export can come later if a specific report format is - required. - ---- - -## 8. Consequences & Open Questions - -- **Which timestamp is authoritative for "the last 2 months"?** `process_start_time` (when the data was *processed*) vs. - `created_at` (when the checkpoint was *reported to Atum*). **Decided (2026-09-25): `process_start_time`** (business - time). For late/backfilled reports the two can differ. -- **Volume at scale.** A long-lived flow accumulates many checkpoints; the proposed index is what keeps a - time-window audit query cheap. -- **Explicit parent -> child edges.** Flow membership gives the downstream lineage *set*; if an audit needs the exact - edge list (who is parent of whom), that is reconstructed today via `get_partitioning_ancestors` per node. A - dedicated "flow graph" (nodes + edges) endpoint is a possible small future addition. -- **Access control** for audit consumers (read-only scopes/authorization) is out of scope here. diff --git a/database/src/main/postgres/flows/V0.8.0.1__get_flow_checkpoints.sql b/database/src/main/postgres/flows/V0.8.0.1__get_flow_checkpoints.sql index 74c90e200..d89c828b6 100644 --- a/database/src/main/postgres/flows/V0.8.0.1__get_flow_checkpoints.sql +++ b/database/src/main/postgres/flows/V0.8.0.1__get_flow_checkpoints.sql @@ -14,8 +14,8 @@ * limitations under the License. */ --- The previous overload is dropped so that the new signature (JSONB multi-value checkpoint properties filter and --- the process start time window) is unambiguous when the function is called with named or defaulted arguments. +-- The previous overload is dropped so that the new signature (JSONB multi-value checkpoint properties filter) is +-- unambiguous when the function is called with named or defaulted arguments. DROP FUNCTION IF EXISTS flows.get_flow_checkpoints(BIGINT, INT, BIGINT, TEXT, HSTORE, BOOLEAN); CREATE OR REPLACE FUNCTION flows.get_flow_checkpoints( @@ -25,8 +25,6 @@ CREATE OR REPLACE FUNCTION flows.get_flow_checkpoints( IN i_checkpoint_name TEXT DEFAULT NULL, IN i_checkpoint_properties JSONB DEFAULT NULL, IN i_latest_first BOOLEAN DEFAULT TRUE, - IN i_from_time TIMESTAMP WITH TIME ZONE DEFAULT NULL, - IN i_to_time TIMESTAMP WITH TIME ZONE DEFAULT NULL, OUT status INTEGER, OUT status_text TEXT, OUT id_checkpoint UUID, @@ -45,9 +43,9 @@ CREATE OR REPLACE FUNCTION flows.get_flow_checkpoints( ) RETURNS SETOF record AS -------------------------------------------------------------------------------------------------------------------- -- --- Function: flows.get_flow_checkpoints(8) +-- Function: flows.get_flow_checkpoints(6) -- Retrieves all checkpoints (measures and their measurement details) related to an input flow (and checkpoint --- name, checkpoint properties and/or process start time window, if specified). +-- name and/or checkpoint properties, if specified). -- -- Note: a single row returned from this function doesn't contain all data related to a single checkpoint - it only -- represents one measure associated with a checkpoint. So even if only a single checkpoint would be retrieved, @@ -63,8 +61,6 @@ CREATE OR REPLACE FUNCTION flows.get_flow_checkpoints( -- property name, have that property with one of the accepted values -- i_latest_first - (optional) if true (default), checkpoints are ordered by process_start_time -- in descending order (latest first); if false, in ascending order --- i_from_time - (optional) if specified, returns only checkpoints with process_start_time >= i_from_time --- i_to_time - (optional) if specified, returns only checkpoints with process_start_time < i_to_time -- -- Note: i_checkpoints_limit and i_offset are used for pagination purposes; -- checkpoints are ordered by process_start_time (descending by default, see i_latest_first) @@ -119,8 +115,6 @@ BEGIN JOIN flows.partitioning_to_flow PF ON C.fk_partitioning = PF.fk_partitioning WHERE PF.fk_flow = i_flow_id AND (i_checkpoint_name IS NULL OR C.checkpoint_name = i_checkpoint_name) - AND (i_from_time IS NULL OR C.process_start_time >= i_from_time) - AND (i_to_time IS NULL OR C.process_start_time < i_to_time) AND ( i_checkpoint_properties IS NULL OR NOT EXISTS ( @@ -161,8 +155,6 @@ BEGIN JOIN flows.partitioning_to_flow PF ON C.fk_partitioning = PF.fk_partitioning WHERE PF.fk_flow = i_flow_id AND (i_checkpoint_name IS NULL OR C.checkpoint_name = i_checkpoint_name) - AND (i_from_time IS NULL OR C.process_start_time >= i_from_time) - AND (i_to_time IS NULL OR C.process_start_time < i_to_time) AND ( i_checkpoint_properties IS NULL OR NOT EXISTS ( @@ -217,5 +209,5 @@ BEGIN END; $$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER; -ALTER FUNCTION flows.get_flow_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE) OWNER TO atum_owner; -GRANT EXECUTE ON FUNCTION flows.get_flow_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE) TO atum_owner; +ALTER FUNCTION flows.get_flow_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN) OWNER TO atum_owner; +GRANT EXECUTE ON FUNCTION flows.get_flow_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN) TO atum_owner; diff --git a/database/src/main/postgres/runs/V0.8.0.2__get_partitioning_checkpoints.sql b/database/src/main/postgres/runs/V0.8.0.2__get_partitioning_checkpoints.sql index 76ad85905..212f5fa28 100644 --- a/database/src/main/postgres/runs/V0.8.0.2__get_partitioning_checkpoints.sql +++ b/database/src/main/postgres/runs/V0.8.0.2__get_partitioning_checkpoints.sql @@ -14,8 +14,8 @@ * limitations under the License. */ --- The previous overload is dropped so that the new signature (JSONB multi-value checkpoint properties filter and --- the process start time window) is unambiguous when the function is called with named or defaulted arguments. +-- The previous overload is dropped so that the new signature (JSONB multi-value checkpoint properties filter) is +-- unambiguous when the function is called with named or defaulted arguments. DROP FUNCTION IF EXISTS runs.get_partitioning_checkpoints(BIGINT, INT, BIGINT, TEXT, HSTORE, BOOLEAN); CREATE OR REPLACE FUNCTION runs.get_partitioning_checkpoints( @@ -25,8 +25,6 @@ CREATE OR REPLACE FUNCTION runs.get_partitioning_checkpoints( IN i_checkpoint_name TEXT DEFAULT NULL, IN i_checkpoint_properties JSONB DEFAULT NULL, IN i_latest_first BOOLEAN DEFAULT TRUE, - IN i_from_time TIMESTAMP WITH TIME ZONE DEFAULT NULL, - IN i_to_time TIMESTAMP WITH TIME ZONE DEFAULT NULL, OUT status INTEGER, OUT status_text TEXT, OUT id_checkpoint UUID, @@ -42,9 +40,9 @@ CREATE OR REPLACE FUNCTION runs.get_partitioning_checkpoints( ) RETURNS SETOF record AS -------------------------------------------------------------------------------------------------------------------- -- --- Function: runs.get_partitioning_checkpoints(8) +-- Function: runs.get_partitioning_checkpoints(6) -- Retrieves all checkpoints (measures and their measurement details) related to an input partitioning (and --- checkpoint name, checkpoint properties and/or process start time window, if specified). +-- checkpoint name and/or checkpoint properties, if specified). -- -- Note: a single row returned from this function doesn't contain all data related to a single checkpoint - it only -- represents one measure associated with a checkpoint. So even if only a single checkpoint would be retrieved, @@ -60,8 +58,6 @@ CREATE OR REPLACE FUNCTION runs.get_partitioning_checkpoints( -- property name, have that property with one of the accepted values -- i_latest_first - (optional) if true (default), checkpoints are ordered by process_start_time -- in descending order (latest first); if false, in ascending order --- i_from_time - (optional) if specified, returns only checkpoints with process_start_time >= i_from_time --- i_to_time - (optional) if specified, returns only checkpoints with process_start_time < i_to_time -- -- Note: i_checkpoints_limit and i_offset are used for pagination purposes; -- checkpoints are ordered by process_start_time (descending by default, see i_latest_first) @@ -106,8 +102,6 @@ BEGIN FROM runs.checkpoints C WHERE C.fk_partitioning = i_partitioning_id AND (i_checkpoint_name IS NULL OR C.checkpoint_name = i_checkpoint_name) - AND (i_from_time IS NULL OR C.process_start_time >= i_from_time) - AND (i_to_time IS NULL OR C.process_start_time < i_to_time) AND ( i_checkpoint_properties IS NULL OR NOT EXISTS ( @@ -146,8 +140,6 @@ BEGIN FROM runs.checkpoints C WHERE C.fk_partitioning = i_partitioning_id AND (i_checkpoint_name IS NULL OR C.checkpoint_name = i_checkpoint_name) - AND (i_from_time IS NULL OR C.process_start_time >= i_from_time) - AND (i_to_time IS NULL OR C.process_start_time < i_to_time) AND ( i_checkpoint_properties IS NULL OR NOT EXISTS ( @@ -197,5 +189,5 @@ BEGIN END; $$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER; -ALTER FUNCTION runs.get_partitioning_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE) OWNER TO atum_owner; -GRANT EXECUTE ON FUNCTION runs.get_partitioning_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE) TO atum_owner; +ALTER FUNCTION runs.get_partitioning_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN) OWNER TO atum_owner; +GRANT EXECUTE ON FUNCTION runs.get_partitioning_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN) TO atum_owner; diff --git a/database/src/main/postgres/runs/V0.8.0.3__checkpoints_partitioning_start_time_index.ddl b/database/src/main/postgres/runs/V0.8.0.3__checkpoints_partitioning_start_time_index.ddl deleted file mode 100644 index 9e7d84004..000000000 --- a/database/src/main/postgres/runs/V0.8.0.3__checkpoints_partitioning_start_time_index.ddl +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - --- Supports the process start time window of runs.get_partitioning_checkpoints and flows.get_flow_checkpoints: --- checkpoints of each partitioning are range-scanned by process_start_time. --- --- Built CONCURRENTLY so writes to runs.checkpoints are not blocked. This file must contain only this statement: --- Flyway then runs it outside of a transaction, which CONCURRENTLY requires. --- If the build fails, it leaves an INVALID index behind; drop it (DROP INDEX CONCURRENTLY runs.checkpoints_idx2) --- and re-run the migration. -CREATE INDEX CONCURRENTLY checkpoints_idx2 ON runs.checkpoints (fk_partitioning, process_start_time); diff --git a/database/src/main/postgres/runs/V0.8.0.4__checkpoint_properties_checkpoint_index.ddl b/database/src/main/postgres/runs/V0.8.0.4__checkpoint_properties_checkpoint_index.ddl deleted file mode 100644 index bf7f54a0f..000000000 --- a/database/src/main/postgres/runs/V0.8.0.4__checkpoint_properties_checkpoint_index.ddl +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - --- Supports the checkpoint properties filter of runs.get_partitioning_checkpoints and flows.get_flow_checkpoints --- (looked up per checkpoint and property name), and the retrieval of a checkpoint's properties. --- --- Built CONCURRENTLY so writes to runs.checkpoint_properties are not blocked. This file must contain only this --- statement: Flyway then runs it outside of a transaction, which CONCURRENTLY requires. --- If the build fails, it leaves an INVALID index behind; drop it --- (DROP INDEX CONCURRENTLY runs.idx_checkpoint_properties_checkpoint_name) and re-run the migration. -CREATE INDEX CONCURRENTLY idx_checkpoint_properties_checkpoint_name ON runs.checkpoint_properties (fk_checkpoint, property_name); diff --git a/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala index f3c71a747..62ad5e6f4 100644 --- a/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala @@ -3,7 +3,7 @@ package za.co.absa.atum.database.flows import io.circe.Json import io.circe.parser.parse import za.co.absa.balta.DBTestSuite -import za.co.absa.balta.classes.{JsonBString, QueryResult} +import za.co.absa.balta.classes.JsonBString import za.co.absa.balta.classes.setter.CustomDBType import java.time.OffsetDateTime @@ -847,148 +847,6 @@ class GetFlowCheckpointsIntegrationTests extends DBTestSuite { } } - test("getFlowCheckpointsV2 should return only checkpoints within the process start time window, across partitionings") { - - val otherPartitioning = JsonBString( - """ - |{ - | "version": 1, - | "keys": ["keyX", "keyY", "keyZ"], - | "keysToValuesMap": { - | "keyX": "value1", - | "keyZ": "value3", - | "keyY": "otherValue2" - | } - |} - |""".stripMargin - ) - - val partitioningIdA: Long = Random.nextLong() - val partitioningIdB: Long = Random.nextLong() - table("runs.partitionings").insert( - add("id_partitioning", partitioningIdA) - .add("partitioning", partitioning) - .add("created_by", "Joseph") - ) - table("runs.partitionings").insert( - add("id_partitioning", partitioningIdB) - .add("partitioning", otherPartitioning) - .add("created_by", "Joseph") - ) - - val flowId: Long = Random.nextLong() - table("flows.flows").insert( - add("id_flow", flowId) - .add("flow_name", "flowNameTimeWindow") - .add("from_pattern", false) - .add("created_by", "Joseph") - .add("fk_primary_partitioning", partitioningIdA) - ) - Seq(partitioningIdA, partitioningIdB).foreach { partitioningId => - table("flows.partitioning_to_flow").insert( - add("fk_flow", flowId) - .add("fk_partitioning", partitioningId) - .add("created_by", "ObviouslySomeTest") - ) - } - - val measureDefinitionIdA: Long = Random.nextLong() - val measureDefinitionIdB: Long = Random.nextLong() - Seq(partitioningIdA -> measureDefinitionIdA, partitioningIdB -> measureDefinitionIdB).foreach { - case (partitioningId, measureDefinitionId) => - table("runs.measure_definitions").insert( - add("id_measure_definition", measureDefinitionId) - .add("fk_partitioning", partitioningId) - .add("measure_name", "cnt") - .add("measured_columns", CustomDBType("""{"col1"}""", "TEXT[]")) - .add("created_by", "Joseph") - ) - } - - // one checkpoint at the start of each month, alternating between the two partitionings of the flow - case class TestCheckpoint(id: UUID, partitioningId: Long, measureDefinitionId: Long, start: OffsetDateTime, executionId: String) - val jan = TestCheckpoint(UUID.randomUUID(), partitioningIdA, measureDefinitionIdA, OffsetDateTime.parse("2026-01-01T00:00:00Z"), "e1") - val feb = TestCheckpoint(UUID.randomUUID(), partitioningIdB, measureDefinitionIdB, OffsetDateTime.parse("2026-02-01T00:00:00Z"), "e1") - val mar = TestCheckpoint(UUID.randomUUID(), partitioningIdA, measureDefinitionIdA, OffsetDateTime.parse("2026-03-01T00:00:00Z"), "e2") - val apr = TestCheckpoint(UUID.randomUUID(), partitioningIdB, measureDefinitionIdB, OffsetDateTime.parse("2026-04-01T00:00:00Z"), "e3") - Seq(jan, feb, mar, apr).foreach { checkpoint => - table("runs.checkpoints").insert( - add("id_checkpoint", checkpoint.id) - .add("fk_partitioning", checkpoint.partitioningId) - .add("checkpoint_name", "CheckpointName") - .add("measured_by_atum_agent", true) - .add("process_start_time", checkpoint.start) - .add("created_by", "Joseph") - ) - table("runs.measurements").insert( - add("fk_measure_definition", checkpoint.measureDefinitionId) - .add("fk_checkpoint", checkpoint.id) - .add("measurement_value", measurementCnt) - ) - table("runs.checkpoint_properties").insert( - add("fk_checkpoint", checkpoint.id) - .add("property_name", "executionID") - .add("property_value", checkpoint.executionId) - ) - } - - def checkpointsOf(queryResult: QueryResult): Seq[(UUID, Long)] = - queryResult.map(row => (row.getUUID("id_checkpoint").get, row.getLong("id_partitioning").get)).toList - - // [feb, apr) spans both partitionings of the flow, latest first - function(fncGetFlowCheckpointsV2) - .setParam("i_flow_id", flowId) - .setParam("i_from_time", feb.start) - .setParam("i_to_time", apr.start) - .execute { queryResult => - assert(checkpointsOf(queryResult) == Seq((mar.id, partitioningIdA), (feb.id, partitioningIdB))) - } - - // the same window, earliest first - function(fncGetFlowCheckpointsV2) - .setParam("i_flow_id", flowId) - .setParam("i_from_time", feb.start) - .setParam("i_to_time", apr.start) - .setParam("i_latest_first", false) - .execute { queryResult => - assert(checkpointsOf(queryResult).map(_._1) == Seq(feb.id, mar.id)) - } - - // combined with a multi-value checkpoint properties filter: executionID IN (e1, e3) since February - function(fncGetFlowCheckpointsV2) - .setParam("i_flow_id", flowId) - .setParam("i_checkpoint_properties", JsonBString("""{"executionID": ["e1", "e3"]}""")) - .setParam("i_from_time", feb.start) - .execute { queryResult => - assert(checkpointsOf(queryResult).map(_._1) == Seq(apr.id, feb.id)) - } - - // paging within the window: has_more is computed over the window only - function(fncGetFlowCheckpointsV2) - .setParam("i_flow_id", flowId) - .setParam("i_checkpoints_limit", 1) - .setParam("i_from_time", feb.start) - .setParam("i_to_time", apr.start) - .execute { queryResult => - val row = queryResult.next() - assert(row.getUUID("id_checkpoint").contains(mar.id)) - assert(row.getBoolean("has_more").contains(true)) - assert(!queryResult.hasNext) - } - function(fncGetFlowCheckpointsV2) - .setParam("i_flow_id", flowId) - .setParam("i_checkpoints_limit", 1) - .setParam("i_offset", 1L) - .setParam("i_from_time", feb.start) - .setParam("i_to_time", apr.start) - .execute { queryResult => - val row = queryResult.next() - assert(row.getUUID("id_checkpoint").contains(feb.id)) - assert(row.getBoolean("has_more").contains(false)) - assert(!queryResult.hasNext) - } - } - private def parseJsonBStringOrThrow(jsonBString: JsonBString): Json = { parse(jsonBString.value).getOrElse(throw new Exception("Failed to parse JsonBString to Json")) } diff --git a/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala index 110a961dd..797b9d77b 100644 --- a/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.database.runs import za.co.absa.balta.DBTestSuite -import za.co.absa.balta.classes.{JsonBString, QueryResult} +import za.co.absa.balta.classes.JsonBString import za.co.absa.balta.classes.setter.CustomDBType import java.time.OffsetDateTime @@ -492,118 +492,4 @@ class GetPartitioningCheckpointsIntegrationTests extends DBTestSuite { } } - test("Returns only checkpoints within the process start time window") { - table("runs.partitionings").insert( - add("partitioning", partitioning1) - .add("created_by", "Daniel") - ) - val fkPartitioning: Long = table("runs.partitionings") - .fieldValue("partitioning", partitioning1, "id_partitioning").get.get - - table("runs.measure_definitions").insert( - add("id_measure_definition", id_measure_definition1) - .add("fk_partitioning", fkPartitioning) - .add("created_by", "Daniel") - .add("measure_name", "measure_1") - .add("measured_columns", measured_columns1) - ) - - // one checkpoint at the start of each month, January to April; named "odd" / "even" by month number - val times = Seq( - OffsetDateTime.parse("2026-01-01T00:00:00Z"), - OffsetDateTime.parse("2026-02-01T00:00:00Z"), - OffsetDateTime.parse("2026-03-01T00:00:00Z"), - OffsetDateTime.parse("2026-04-01T00:00:00Z") - ) - val checkpointIds = times.map(_ => UUID.randomUUID()) - times.zip(checkpointIds).zipWithIndex.foreach { case ((time, checkpointId), i) => - table("runs.checkpoints").insert( - add("id_checkpoint", checkpointId) - .add("fk_partitioning", fkPartitioning) - .add("checkpoint_name", if (i % 2 == 0) "odd" else "even") - .add("process_start_time", time) - .add("process_end_time", endTime) - .add("measured_by_atum_agent", true) - .add("created_by", "Daniel") - ) - table("runs.measurements").insert( - add("fk_checkpoint", checkpointId) - .add("fk_measure_definition", id_measure_definition1) - .add("measurement_value", measurement1) - ) - } - val Seq(jan, feb, mar, apr) = checkpointIds - - def checkpointsOf(queryResult: QueryResult): Seq[UUID] = - queryResult.map(_.getUUID("id_checkpoint").get).toList - - // i_from_time is inclusive - function(fncGetPartitioningCheckpoints) - .setParam("i_partitioning_id", fkPartitioning) - .setParam("i_from_time", times(1)) - .execute { queryResult => - assert(checkpointsOf(queryResult) == Seq(apr, mar, feb)) - } - - // i_to_time is exclusive - function(fncGetPartitioningCheckpoints) - .setParam("i_partitioning_id", fkPartitioning) - .setParam("i_to_time", times(2)) - .execute { queryResult => - assert(checkpointsOf(queryResult) == Seq(feb, jan)) - } - - // both bounds, earliest first - function(fncGetPartitioningCheckpoints) - .setParam("i_partitioning_id", fkPartitioning) - .setParam("i_from_time", times(1)) - .setParam("i_to_time", times(3)) - .setParam("i_latest_first", false) - .execute { queryResult => - assert(checkpointsOf(queryResult) == Seq(feb, mar)) - } - - // paging within the window: has_more is computed over the window only - function(fncGetPartitioningCheckpoints) - .setParam("i_partitioning_id", fkPartitioning) - .setParam("i_checkpoints_limit", 1) - .setParam("i_from_time", times(1)) - .setParam("i_to_time", times(3)) - .execute { queryResult => - val row = queryResult.next() - assert(row.getUUID("id_checkpoint").contains(mar)) - assert(row.getBoolean("has_more").contains(true)) - assert(!queryResult.hasNext) - } - function(fncGetPartitioningCheckpoints) - .setParam("i_partitioning_id", fkPartitioning) - .setParam("i_checkpoints_limit", 1) - .setParam("i_offset", 1L) - .setParam("i_from_time", times(1)) - .setParam("i_to_time", times(3)) - .execute { queryResult => - val row = queryResult.next() - assert(row.getUUID("id_checkpoint").contains(feb)) - assert(row.getBoolean("has_more").contains(false)) - assert(!queryResult.hasNext) - } - - // combined with the checkpoint name filter - function(fncGetPartitioningCheckpoints) - .setParam("i_partitioning_id", fkPartitioning) - .setParam("i_checkpoint_name", "odd") - .setParam("i_from_time", times(1)) - .execute { queryResult => - assert(checkpointsOf(queryResult) == Seq(mar)) - } - - // a window without checkpoints returns no rows - function(fncGetPartitioningCheckpoints) - .setParam("i_partitioning_id", fkPartitioning) - .setParam("i_from_time", OffsetDateTime.parse("2027-01-01T00:00:00Z")) - .execute { queryResult => - assert(!queryResult.hasNext) - } - } - } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala index f4f8dd8f9..4e41985ec 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala @@ -45,12 +45,12 @@ case class FlowReader[F[_]](mainFlowPartitioning: AtumPartitions)(implicit /** * Function to retrieve a page of checkpoints belonging to the flow, optionally filtered. - * The checkpoints are ordered by their process start time, latest first unless the filter says otherwise. + * The checkpoints are ordered by their process start time, latest first. * * @param pageSize - the size of the page (record count) to be returned * @param offset - offset of the page (starting position) * @param includeProperties - whether to include checkpoint properties in the response - * @param filter - the checkpoints to return (name, properties, process start time window) and their order + * @param filter - the checkpoints to return (by name and/or properties) * @return - a page of checkpoints */ def getCheckpointsPage( @@ -68,32 +68,6 @@ case class FlowReader[F[_]](mainFlowPartitioning: AtumPartitions)(implicit } yield checkpointsOrError } - /** - * Function to retrieve all checkpoints belonging to the flow that satisfy the filter, querying them page by page. - * The checkpoints are ordered by their process start time, latest first unless the filter says otherwise. - * - * All matching checkpoints are held in memory, so bound the query with the filter, typically with a process start - * time window, e.g. `CheckpointFilter(from = Some(ZonedDateTime.now().minusMonths(2)))` for the last two months. - * - * @param filter - the checkpoints to return (name, properties, process start time window) and their order - * @param includeProperties - whether to include checkpoint properties in the response - * @param pageSize - the size of the pages (record count) to query the checkpoints in - * @return - all the checkpoints satisfying the filter, or the first error encountered - */ - def getAllCheckpoints( - filter: CheckpointFilter = CheckpointFilter.empty, - includeProperties: Boolean = false, - pageSize: Int = 100 - ): F[RequestResult[Seq[CheckpointWithPartitioningDTO]]] = { - for { - flowIdOrError <- flowId - checkpointsOrError <- mapRequestResultF( - flowIdOrError, - (flowId: Long) => queryAllPages(pageSize, queryCheckpoints(flowId, filter, _, _, includeProperties)) - ) - } yield checkpointsOrError - } - /** * Function to retrieve a page of checkpoints of the given name belonging to the flow. * The checkpoints are ordered by their process start time, latest first. diff --git a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala index 6f3dd89af..0346788cc 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala @@ -43,12 +43,12 @@ case class PartitioningReader[F[_]](partitioning: AtumPartitions)(implicit /** * Function to retrieve a page of checkpoints belonging to the partitioning, optionally filtered. - * The checkpoints are ordered by their process start time, latest first unless the filter says otherwise. + * The checkpoints are ordered by their process start time, latest first. * * @param pageSize - the size of the page (record count) to be returned * @param offset - offset of the page (starting position) * @param includeProperties - whether to include checkpoint properties in the response - * @param filter - the checkpoints to return (name, properties, process start time window) and their order + * @param filter - the checkpoints to return (by name and/or properties) * @return - a page of checkpoints */ def getCheckpointsPage( @@ -66,33 +66,6 @@ case class PartitioningReader[F[_]](partitioning: AtumPartitions)(implicit } yield checkpointsOrError } - /** - * Function to retrieve all checkpoints belonging to the partitioning that satisfy the filter, querying them page by - * page. The checkpoints are ordered by their process start time, latest first unless the filter says otherwise. - * - * All matching checkpoints are held in memory, so bound the query with the filter, typically with a process start - * time window, e.g. `CheckpointFilter(from = Some(ZonedDateTime.now().minusMonths(2)))` for the last two months. - * - * @param filter - the checkpoints to return (name, properties, process start time window) and their order - * @param includeProperties - whether to include checkpoint properties in the response - * @param pageSize - the size of the pages (record count) to query the checkpoints in - * @return - all the checkpoints satisfying the filter, or the first error encountered - */ - def getAllCheckpoints( - filter: CheckpointFilter = CheckpointFilter.empty, - includeProperties: Boolean = false, - pageSize: Int = 100 - ): F[RequestResult[Seq[CheckpointV2DTO]]] = { - for { - partitioningIdOrError <- partitioningId(partitioning) - checkpointsOrError <- mapRequestResultF( - partitioningIdOrError, - (partitioningId: Long) => - queryAllPages(pageSize, queryCheckpoints(partitioningId, filter, _, _, includeProperties)) - ) - } yield checkpointsOrError - } - /** * Function to retrieve a page of checkpoints of the given name belonging to the partitioning. (While the usual logic * would suggest, there would be only one checkpoint of a name, nothing prevents to have checkpoints of the same name; diff --git a/reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala b/reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala index 393312fb9..46b80ebd9 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala @@ -22,7 +22,6 @@ import sttp.client3.circe.asJson import sttp.model.Uri import sttp.monad.MonadError import sttp.monad.syntax._ -import za.co.absa.atum.model.envelopes.SuccessResponse.PaginatedResponse import za.co.absa.atum.reader.core.RequestResult._ import za.co.absa.atum.reader.server.ServerConfig import za.co.absa.atum.reader.exceptions.RequestException.CirceError @@ -49,27 +48,6 @@ abstract class Reader[F[_]](implicit case Left(a) => me.unit(Left(a)) } - /** - * Queries the pages one after another, starting at offset 0, until the server reports there is no more data. - * - * @param pageSize - the size of the page (record count) to query - * @param queryPage - function querying a page of the given size (limit) at the given offset - * @return - the records of all the pages, in order, or the first error encountered - */ - protected def queryAllPages[T]( - pageSize: Int, - queryPage: (Int, Long) => F[RequestResult[PaginatedResponse[T]]] - ): F[RequestResult[Seq[T]]] = { - def queryFrom(offset: Long, collected: Vector[T]): F[RequestResult[Seq[T]]] = { - queryPage(pageSize, offset).flatMap { - case Right(page) if page.pagination.hasMore => queryFrom(offset + pageSize, collected ++ page.data) - case Right(page) => me.unit(RequestOK[Seq[T]](collected ++ page.data)) - case Left(error) => me.unit(RequestFail[Seq[T]](error)) - } - } - queryFrom(0, Vector.empty) - } - protected def getQuery[R: Decoder]( endpointUri: String, params: Map[String, String] = Map.empty diff --git a/reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala b/reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala index 33b8e93c2..5ff07c8ff 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala @@ -18,44 +18,30 @@ package za.co.absa.atum.reader.requests import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ -import java.time.ZonedDateTime - /** * Filter of a checkpoints query. All its parts are optional and a checkpoint has to satisfy all the given ones. * - * Example - checkpoints of two executions processed in the last two months, latest first: + * Example - checkpoints of the given name, of any of the two executions: * {{{ * CheckpointFilter( - * properties = Map("executionID" -> Set("a", "b")), - * from = Some(ZonedDateTime.now().minusMonths(2)) + * name = Some("Data written"), + * properties = Map("executionID" -> Set("a", "b")) * ) * }}} * - * Note: the time window and properties with more than one value require an Atum server supporting them; older servers - * ignore the time window and reject multi-value properties. + * Note: properties with more than one value require an Atum server supporting them; older servers reject them. * - * @param name - only checkpoints of this name - * @param properties - only checkpoints that, for every property name given, have that property with one of the given - * values, e.g. `Map("executionID" -> Set("a", "b"))` means `executionID IN (a, b)` - * @param from - only checkpoints with process start time at or after this time (inclusive) - * @param to - only checkpoints with process start time before this time (exclusive) - * @param latestFirst - order of the checkpoints by their process start time: latest first if `true`, earliest first if - * `false`, server default (latest first) if not specified + * @param name - only checkpoints of this name + * @param properties - only checkpoints that, for every property name given, have that property with one of the given + * values, e.g. `Map("executionID" -> Set("a", "b"))` means `executionID IN (a, b)` */ case class CheckpointFilter( name: Option[String] = None, - properties: Map[String, Set[String]] = Map.empty, - from: Option[ZonedDateTime] = None, - to: Option[ZonedDateTime] = None, - latestFirst: Option[Boolean] = None + properties: Map[String, Set[String]] = Map.empty ) { private[reader] def toQueryParams: Map[String, String] = { - name.map(QueryParamNames.CheckpointName -> _).toMap ++ - propertiesParam ++ - from.map(QueryParamNames.From -> _.toInstant.toString) ++ - to.map(QueryParamNames.To -> _.toInstant.toString) ++ - latestFirst.map(QueryParamNames.LatestFirst -> _.toString) + name.map(QueryParamNames.CheckpointName -> _).toMap ++ propertiesParam } private def propertiesParam: Option[(String, String)] = { diff --git a/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala b/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala index e17baddcd..f1bf111f9 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala @@ -22,7 +22,4 @@ object QueryParamNames { final val CheckpointName = "checkpoint-name" final val CheckpointProperties = "checkpoint-properties" final val IncludeProperties = "include-properties" - final val LatestFirst = "latest-first" - final val From = "from" - final val To = "to" } diff --git a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala index df0ad5b49..dc918b00d 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala @@ -21,7 +21,6 @@ import sttp.capabilities import sttp.client3.monad.IdMonad import sttp.client3.{Identity, Response, SttpBackend} import sttp.client3.testing.SttpBackendStub -import sttp.model.StatusCode import sttp.model.Uri.QuerySegment.KeyValue import sttp.monad.MonadError import za.co.absa.atum.model.ResultValueType @@ -308,7 +307,7 @@ class FlowReaderUnitTests extends AnyFunSuiteLike { assert(result.isRight) } - test("The flow checkpoints are queried with name, multi-value properties and time window in a single request") { + test("The flow checkpoints are queried with name and multi-value properties in a single request") { implicit val server: SttpBackendStub[Identity, capabilities.WebSockets] = SttpBackendStub.synchronous .whenRequestMatchesPartial { case r if r.uri.path.endsWith(List("partitionings")) => @@ -321,71 +320,18 @@ class FlowReaderUnitTests extends AnyFunSuiteLike { assert(r.uri.querySegments.contains(KeyValue("include-properties", "true"))) assert(r.uri.querySegments.contains(KeyValue("checkpoint-name", "Test checkpoints 1"))) assert(r.uri.querySegments.contains(KeyValue("checkpoint-properties", multiValuePropertiesEncoded))) - assert(r.uri.querySegments.contains(KeyValue("from", "2026-06-01T00:00:00Z"))) - assert(r.uri.querySegments.contains(KeyValue("to", "2026-08-01T00:00:00Z"))) - assert(r.uri.querySegments.contains(KeyValue("latest-first", "false"))) Response.ok(checkpointsResponse) } val filter = CheckpointFilter( name = Some("Test checkpoints 1"), - properties = Map("executionID" -> Set("id2", "id1")), - from = Some(ZonedDateTime.parse("2026-06-01T02:00:00+02:00")), - to = Some(ZonedDateTime.parse("2026-08-01T00:00:00Z")), - latestFirst = Some(false) + properties = Map("executionID" -> Set("id2", "id1")) ) val result = FlowReader(AtumPartitions(List("a" -> "b", "c" -> "d"))) .getCheckpointsPage(pageSize = 5, offset = 10, includeProperties = true, filter = filter) assert(result.map(_.data.size) == Right(2)) } - test("All flow checkpoints satisfying the filter are queried page by page") { - var flowIdQueries = 0 - var queriedOffsets = Vector.empty[String] - implicit val server: SttpBackendStub[Identity, capabilities.WebSockets] = SttpBackendStub.synchronous - .whenRequestMatchesPartial { - case r if r.uri.path.endsWith(List("partitionings")) => - Response.ok(partitioningResponse) - case r if r.uri.path.endsWith(List("partitionings", "7", "main-flow")) => - flowIdQueries += 1 - Response.ok(flowResponse) - case r if r.uri.path.endsWith(List("flows", "42", "checkpoints")) => - assert(r.uri.querySegments.contains(KeyValue("limit", "2"))) - assert(r.uri.querySegments.contains(KeyValue("from", "2026-06-01T00:00:00Z"))) - val offset = r.uri.paramsMap("offset") - queriedOffsets :+= offset - if (offset == "0") Response.ok(checkpointsResponseHasMore) else Response.ok(checkpointsResponse) - } - - val filter = CheckpointFilter(from = Some(ZonedDateTime.parse("2026-06-01T00:00:00Z"))) - val result = FlowReader(AtumPartitions(List("a" -> "b", "c" -> "d"))).getAllCheckpoints(filter, pageSize = 2) - - assert(result.map(_.size) == Right(4)) - assert(queriedOffsets == Vector("0", "2")) - assert(flowIdQueries == 1) - } - - test("Querying all flow checkpoints stops at the first failing page") { - var queriedOffsets = Vector.empty[String] - implicit val server: SttpBackendStub[Identity, capabilities.WebSockets] = SttpBackendStub.synchronous - .whenRequestMatchesPartial { - case r if r.uri.path.endsWith(List("partitionings")) => - Response.ok(partitioningResponse) - case r if r.uri.path.endsWith(List("partitionings", "7", "main-flow")) => - Response.ok(flowResponse) - case r if r.uri.path.endsWith(List("flows", "42", "checkpoints")) => - val offset = r.uri.paramsMap("offset") - queriedOffsets :+= offset - if (offset == "0") Response.ok(checkpointsResponseHasMore) - else Response(internalServerErrorResponse, StatusCode.InternalServerError) - } - - val result = FlowReader(AtumPartitions(List("a" -> "b", "c" -> "d"))).getAllCheckpoints(pageSize = 2) - - assert(result.isLeft) - assert(queriedOffsets == Vector("0", "2")) - } - test("Instantiate FlowReader with implicit arguments passed explicitly") { val atumPartitions: AtumPartitions = AtumPartitions(List( "a" -> "b", @@ -422,9 +368,6 @@ object FlowReaderUnitTests { // base64url-encoded {"executionID":["id1","id2"]} private val multiValuePropertiesEncoded = "eyJleGVjdXRpb25JRCI6WyJpZDEiLCJpZDIiXX0=" - private val internalServerErrorResponse = - """{"message": "boom", "requestId": "29ce91a7-b668-41d2-a160-26402551fb0b"}""" - private val partitioningEncoded = "W3sia2V5IjoiYSIsInZhbHVlIjoiYiJ9LHsia2V5IjoiYyIsInZhbHVlIjoiZCJ9XQ==" private val partitioningResponse = @@ -542,8 +485,6 @@ object FlowReaderUnitTests { |} |""".stripMargin - private val checkpointsResponseHasMore = checkpointsResponse.replace("\"hasMore\" : false", "\"hasMore\" : true") - private val checkpointsResponseWithProperties = """ |{ diff --git a/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala index ceb25edb6..2226e4c3e 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala @@ -277,7 +277,7 @@ class PartitioningReaderUnitTests extends AnyFunSuiteLike { assert(result.isRight) } - test("The partitioning checkpoints are queried with name, multi-value properties and time window in a single request") { + test("The partitioning checkpoints are queried with name and multi-value properties in a single request") { implicit val server: SttpBackendStub[Identity, capabilities.WebSockets] = SttpBackendStub.synchronous .whenRequestMatchesPartial { case r if r.uri.path.endsWith(List(V2Paths.Partitionings)) => @@ -288,43 +288,17 @@ class PartitioningReaderUnitTests extends AnyFunSuiteLike { assert(r.uri.querySegments.contains(KeyValue("checkpoint-name", "Test checkpoints 1"))) // base64url-encoded {"executionID":["id1","id2"]} assert(r.uri.querySegments.contains(KeyValue("checkpoint-properties", "eyJleGVjdXRpb25JRCI6WyJpZDEiLCJpZDIiXX0="))) - assert(r.uri.querySegments.contains(KeyValue("from", "2026-06-01T00:00:00Z"))) - assert(!r.uri.paramsMap.contains("to")) - assert(!r.uri.paramsMap.contains("latest-first")) Response.ok(checkpointsResponse) } val filter = CheckpointFilter( name = Some("Test checkpoints 1"), - properties = Map("executionID" -> Set("id1", "id2")), - from = Some(ZonedDateTime.parse("2026-06-01T00:00:00Z")) + properties = Map("executionID" -> Set("id1", "id2")) ) val result = PartitioningReader(AtumPartitions(List("a" -> "b", "c" -> "d"))).getCheckpointsPage(filter = filter) assert(result.map(_.data.size) == Right(2)) } - test("All partitioning checkpoints satisfying the filter are queried page by page") { - var queriedOffsets = Vector.empty[String] - implicit val server: SttpBackendStub[Identity, capabilities.WebSockets] = SttpBackendStub.synchronous - .whenRequestMatchesPartial { - case r if r.uri.path.endsWith(List(V2Paths.Partitionings)) => - Response.ok(partitioningResponse) - case r if r.uri.path.endsWith(List(V2Paths.Partitionings, "7", V2Paths.Checkpoints)) => - assert(r.uri.querySegments.contains(KeyValue("limit", "2"))) - assert(r.uri.querySegments.contains(KeyValue("checkpoint-name", "Test checkpoints 1"))) - val offset = r.uri.paramsMap("offset") - queriedOffsets :+= offset - if (offset == "4") Response.ok(checkpointsResponse) - else Response.ok(checkpointsResponse.replace("\"hasMore\" : false", "\"hasMore\" : true")) - } - - val result = PartitioningReader(AtumPartitions(List("a" -> "b", "c" -> "d"))) - .getAllCheckpoints(CheckpointFilter(name = Some("Test checkpoints 1")), pageSize = 2) - - assert(result.map(_.size) == Right(6)) - assert(queriedOffsets == Vector("0", "2", "4")) - } - test("The partitioning additional data are properly queried and delivered as DTO") { val atumPartitions: AtumPartitions = AtumPartitions( List( diff --git a/reader/src/test/scala/za/co/absa/atum/reader/requests/CheckpointFilterUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/requests/CheckpointFilterUnitTests.scala index e40635059..93b0cbad4 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/requests/CheckpointFilterUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/requests/CheckpointFilterUnitTests.scala @@ -19,8 +19,6 @@ package za.co.absa.atum.reader.requests import org.scalatest.funsuite.AnyFunSuiteLike import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ -import java.time.ZonedDateTime - class CheckpointFilterUnitTests extends AnyFunSuiteLike { test("An empty filter adds no query params") { @@ -45,19 +43,13 @@ class CheckpointFilterUnitTests extends AnyFunSuiteLike { ) } - test("The time window is sent as UTC instants and all filter parts are combined") { - val filter = CheckpointFilter( - name = Some("checkpoint name"), - from = Some(ZonedDateTime.parse("2026-06-01T02:00:00+02:00[Europe/Budapest]")), - to = Some(ZonedDateTime.parse("2026-08-01T00:00:00.5Z")), - latestFirst = Some(false) - ) + test("Name and properties are combined") { + val filter = CheckpointFilter(name = Some("checkpoint name"), properties = Map("env" -> Set("prod"))) assert( filter.toQueryParams == Map( "checkpoint-name" -> "checkpoint name", - "from" -> "2026-06-01T00:00:00Z", - "to" -> "2026-08-01T00:00:00.500Z", - "latest-first" -> "false" + // base64url-encoded {"env":"prod"} + "checkpoint-properties" -> "eyJlbnYiOiJwcm9kIn0=" ) ) } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala index b04dc1f8b..2534988f7 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala @@ -35,8 +35,6 @@ import doobie.postgres.implicits._ import za.co.absa.atum.server.model.database.CheckpointItemWithPartitioningFromDB -import java.time.ZonedDateTime - class GetFlowCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieMultipleResultFunctionWithAggStatus[GetFlowCheckpointsArgs, Option[ CheckpointItemWithPartitioningFromDB @@ -46,10 +44,7 @@ class GetFlowCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task] fr"${input.limit}", fr"${input.offset}", fr"${input.checkpointName}", - fr"${input.checkpointProperties.map(_.asJson)}", - fr"${input.latestFirst}", - fr"${input.from}", - fr"${input.to}" + fr"${input.checkpointProperties.map(_.asJson)}" ) ) with StandardStatusHandling @@ -78,10 +73,7 @@ object GetFlowCheckpoints { limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, Seq[String]]], - latestFirst: Option[Boolean], - from: Option[ZonedDateTime], - to: Option[ZonedDateTime] + checkpointProperties: Option[Map[String, Seq[String]]] ) val layer: URLayer[PostgresDatabaseProvider, GetFlowCheckpoints] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala index 0632ac744..b43ec3662 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala @@ -35,8 +35,6 @@ import io.circe.generic.auto._ import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstRowStatusAggregator import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling -import java.time.ZonedDateTime - class GetPartitioningCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieMultipleResultFunctionWithAggStatus[GetPartitioningCheckpointsArgs, Option[ CheckpointItemFromDB.Paginated @@ -47,9 +45,7 @@ class GetPartitioningCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngi fr"${args.offset}", fr"${args.checkpointName}", fr"${args.checkpointProperties.map(_.asJson)}", - fr"${args.latestFirst}", - fr"${args.from}", - fr"${args.to}" + fr"${args.latestFirst}" ) ) with StandardStatusHandling @@ -76,9 +72,7 @@ object GetPartitioningCheckpoints { offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], - latestFirst: Option[Boolean], - from: Option[ZonedDateTime], - to: Option[ZonedDateTime] + latestFirst: Option[Boolean] ) val layer: URLayer[PostgresDatabaseProvider, GetPartitioningCheckpoints] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointController.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointController.scala index 4d521b999..428f55a1e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointController.scala @@ -22,7 +22,6 @@ import za.co.absa.atum.model.envelopes.SuccessResponse.{PaginatedResponse, Singl import zio.IO import zio.macros.accessible -import java.time.ZonedDateTime import java.util.UUID @accessible @@ -46,8 +45,6 @@ trait CheckpointController { checkpointName: Option[String] = None, checkpointProperties: Option[Map[String, Seq[String]]] = None, latestFirst: Option[Boolean] = None, - from: Option[ZonedDateTime] = None, - to: Option[ZonedDateTime] = None, includeProperties: Boolean ): IO[ErrorResponse, PaginatedResponse[CheckpointV2DTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerImpl.scala index 464cf05ec..46897914e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerImpl.scala @@ -25,7 +25,6 @@ import za.co.absa.atum.server.api.v2.service.CheckpointService import za.co.absa.atum.server.model.PaginatedResult import zio._ -import java.time.ZonedDateTime import java.util.UUID class CheckpointControllerImpl(checkpointService: CheckpointService) extends CheckpointController with BaseController { @@ -67,8 +66,6 @@ class CheckpointControllerImpl(checkpointService: CheckpointService) extends Che checkpointName: Option[String] = None, checkpointProperties: Option[Map[String, Seq[String]]] = None, latestFirst: Option[Boolean] = None, - from: Option[ZonedDateTime] = None, - to: Option[ZonedDateTime] = None, includeProperties: Boolean ): IO[ErrorResponse, PaginatedResponse[CheckpointV2DTO]] = { mapToPaginatedResponse( @@ -82,8 +79,6 @@ class CheckpointControllerImpl(checkpointService: CheckpointService) extends Che checkpointName, checkpointProperties, latestFirst, - from, - to, includeProperties ) ) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowController.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowController.scala index 90cca4761..23be150c6 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowController.scala @@ -22,8 +22,6 @@ import za.co.absa.atum.model.envelopes.SuccessResponse.PaginatedResponse import zio.IO import zio.macros.accessible -import java.time.ZonedDateTime - @accessible trait FlowController { @@ -33,9 +31,6 @@ trait FlowController { offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], - latestFirst: Option[Boolean], - from: Option[ZonedDateTime], - to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ErrorResponse, PaginatedResponse[CheckpointWithPartitioningDTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerImpl.scala index 3d1419de0..8a031b0f4 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerImpl.scala @@ -24,8 +24,6 @@ import za.co.absa.atum.server.api.v2.service.FlowService import za.co.absa.atum.server.model.PaginatedResult import zio._ -import java.time.ZonedDateTime - class FlowControllerImpl(flowService: FlowService) extends FlowController with BaseController { // to be replaced (and moved to checkpointcontroller) with new implementation in #233 @@ -35,24 +33,11 @@ class FlowControllerImpl(flowService: FlowService) extends FlowController with B offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], - latestFirst: Option[Boolean], - from: Option[ZonedDateTime], - to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ErrorResponse, PaginatedResponse[CheckpointWithPartitioningDTO]] = { val flowData = serviceCall[PaginatedResult[CheckpointWithPartitioningDTO], PaginatedResult[CheckpointWithPartitioningDTO]]( - flowService.getFlowCheckpoints( - flowId, - limit, - offset, - checkpointName, - checkpointProperties, - latestFirst, - from, - to, - includeProperties - ) + flowService.getFlowCheckpoints(flowId, limit, offset, checkpointName, checkpointProperties, includeProperties) ) mapToPaginatedResponse(limit, offset, flowData) } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala index 99baf6be9..906a22cc0 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala @@ -20,7 +20,7 @@ import sttp.model.StatusCode import sttp.tapir.generic.auto.schemaForCaseClass import sttp.tapir.json.circe.jsonBody import sttp.tapir.ztapir._ -import sttp.tapir.{Codec, CodecFormat, DecodeResult, EndpointInput, PublicEndpoint, ValidationResult, Validator} +import sttp.tapir.{Codec, CodecFormat, DecodeResult, PublicEndpoint, Validator} import za.co.absa.atum.model.ApiPaths._ import za.co.absa.atum.model.dto._ import za.co.absa.atum.model.envelopes.ErrorResponse @@ -29,7 +29,6 @@ import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ import za.co.absa.atum.server.api.v2.controller.{CheckpointController, FlowController, PartitioningController} import za.co.absa.atum.server.api.common.http.{BaseEndpoints, HttpEnv} -import java.time.ZonedDateTime import java.util.UUID import scala.util.{Failure, Success, Try} @@ -50,21 +49,6 @@ object Endpoints extends BaseEndpoints { } }(_.asBase64EncodedJsonString) - // Optional half-open window `from <= process start time < to` for checkpoint queries, supplied as ISO-8601 date-times - // with an offset, e.g. `from=2026-06-01T00:00:00Z`. A malformed value, or `from` not before `to`, results in a - // 400 Bad Request. - private val checkpointTimeWindowInput: EndpointInput[(Option[ZonedDateTime], Option[ZonedDateTime])] = - query[Option[ZonedDateTime]]("from") - .description("inclusive lower bound of the checkpoint's process start time (ISO-8601 date-time with offset)") - .and( - query[Option[ZonedDateTime]]("to") - .description("exclusive upper bound of the checkpoint's process start time (ISO-8601 date-time with offset)") - ) - .validate(Validator.custom { - case (Some(from), Some(to)) if !from.isBefore(to) => ValidationResult.Invalid("'from' must be before 'to'") - case _ => ValidationResult.Valid - }) - val postCheckpointEndpoint : PublicEndpoint[(Long, CheckpointV2DTO), ErrorResponse, (SingleSuccessResponse[CheckpointV2DTO], String), Any] = { apiV2.post @@ -133,7 +117,7 @@ object Endpoints extends BaseEndpoints { } val getPartitioningCheckpointsEndpoint - : PublicEndpoint[(Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Option[ZonedDateTime], Option[ZonedDateTime], Boolean), ErrorResponse, PaginatedResponse[ + : PublicEndpoint[(Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Boolean), ErrorResponse, PaginatedResponse[ CheckpointV2DTO ], Any] = { apiV2.get @@ -144,7 +128,6 @@ object Endpoints extends BaseEndpoints { .in(query[Option[String]]("checkpoint-name")) .in(query[Option[Map[String, Seq[String]]]]("checkpoint-properties")) .in(query[Option[Boolean]]("latest-first")) - .in(checkpointTimeWindowInput) .in(query[Boolean]("include-properties").default(false)) .out(statusCode(StatusCode.Ok)) .out(jsonBody[PaginatedResponse[CheckpointV2DTO]]) @@ -152,7 +135,7 @@ object Endpoints extends BaseEndpoints { } val getFlowCheckpointsEndpoint - : PublicEndpoint[(Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Option[ZonedDateTime], Option[ZonedDateTime], Boolean), ErrorResponse, PaginatedResponse[ + : PublicEndpoint[(Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Boolean), ErrorResponse, PaginatedResponse[ CheckpointWithPartitioningDTO ], Any] = { apiV2.get @@ -161,8 +144,6 @@ object Endpoints extends BaseEndpoints { .in(query[Long]("offset").default(0L).validate(Validator.min(0L))) .in(query[Option[String]]("checkpoint-name")) .in(query[Option[Map[String, Seq[String]]]]("checkpoint-properties")) - .in(query[Option[Boolean]]("latest-first")) - .in(checkpointTimeWindowInput) .in(query[Boolean]("include-properties").default(false)) .out(statusCode(StatusCode.Ok)) .out(jsonBody[PaginatedResponse[CheckpointWithPartitioningDTO]]) @@ -271,23 +252,23 @@ object Endpoints extends BaseEndpoints { } ), createServerEndpoint[ - (Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Option[ZonedDateTime], Option[ZonedDateTime], Boolean), + (Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Boolean), ErrorResponse, PaginatedResponse[CheckpointV2DTO] ]( getPartitioningCheckpointsEndpoint, - { case (partitioningId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], from: Option[ZonedDateTime], to: Option[ZonedDateTime], includeProperties: Boolean) => - CheckpointController.getPartitioningCheckpoints(partitioningId, limit, offset, checkpointName, checkpointProperties, latestFirst, from, to, includeProperties) + { case (partitioningId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], includeProperties: Boolean) => + CheckpointController.getPartitioningCheckpoints(partitioningId, limit, offset, checkpointName, checkpointProperties, latestFirst, includeProperties) } ), createServerEndpoint[ - (Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Option[ZonedDateTime], Option[ZonedDateTime], Boolean), + (Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Boolean), ErrorResponse, PaginatedResponse[CheckpointWithPartitioningDTO] ]( getFlowCheckpointsEndpoint, - { case (flowId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], from: Option[ZonedDateTime], to: Option[ZonedDateTime], includeProperties: Boolean) => - FlowController.getFlowCheckpoints(flowId, limit, offset, checkpointName, checkpointProperties, latestFirst, from, to, includeProperties) + { case (flowId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], includeProperties: Boolean) => + FlowController.getFlowCheckpoints(flowId, limit, offset, checkpointName, checkpointProperties, includeProperties) } ), createServerEndpoint(getPartitioningByIdEndpoint, PartitioningController.getPartitioningById), diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepository.scala index 018cf3df6..14856ecf2 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepository.scala @@ -22,7 +22,6 @@ import za.co.absa.atum.server.model.PaginatedResult import zio._ import zio.macros.accessible -import java.time.ZonedDateTime import java.util.UUID @accessible @@ -40,8 +39,6 @@ trait CheckpointRepository { checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], - from: Option[ZonedDateTime], - to: Option[ZonedDateTime], includeProperties: Boolean ): IO[DatabaseError, PaginatedResult[CheckpointV2DTO]] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryImpl.scala index d1c13b425..3a8cb1c22 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryImpl.scala @@ -31,7 +31,6 @@ import za.co.absa.atum.server.model.database.CheckpointItemFromDB import zio._ import zio.interop.catz.asyncInstance -import java.time.ZonedDateTime import java.util.UUID class CheckpointRepositoryImpl( @@ -78,22 +77,11 @@ class CheckpointRepositoryImpl( checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], - from: Option[ZonedDateTime], - to: Option[ZonedDateTime], includeProperties: Boolean ): IO[DatabaseError, PaginatedResult[CheckpointV2DTO]] = { dbMultipleResultCallWithAggregatedStatus( getPartitioningCheckpointsFn( - GetPartitioningCheckpointsArgs( - partitioningId, - limit, - offset, - checkpointName, - checkpointProperties, - latestFirst, - from, - to - ) + GetPartitioningCheckpointsArgs(partitioningId, limit, offset, checkpointName, checkpointProperties, latestFirst) ), "getPartitioningCheckpoints" ) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepository.scala index 15e9b9951..d47df8ff1 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepository.scala @@ -22,8 +22,6 @@ import za.co.absa.atum.server.model.PaginatedResult import zio._ import zio.macros.accessible -import java.time.ZonedDateTime - @accessible trait FlowRepository { @@ -33,9 +31,6 @@ trait FlowRepository { offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], - latestFirst: Option[Boolean], - from: Option[ZonedDateTime], - to: Option[ZonedDateTime], includeProperties: Boolean ): IO[DatabaseError, PaginatedResult[CheckpointWithPartitioningDTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryImpl.scala index d861b046d..b668384be 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryImpl.scala @@ -29,8 +29,6 @@ import za.co.absa.atum.server.model.database.CheckpointItemWithPartitioningFromD import zio._ import zio.interop.catz.asyncInstance -import java.time.ZonedDateTime - class FlowRepositoryImpl( getFlowCheckpointsFn: GetFlowCheckpoints, override val getCheckpointPropertiesFn: GetCheckpointProperties @@ -42,15 +40,10 @@ class FlowRepositoryImpl( offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], - latestFirst: Option[Boolean], - from: Option[ZonedDateTime], - to: Option[ZonedDateTime], includeProperties: Boolean ): IO[DatabaseError, PaginatedResult[CheckpointWithPartitioningDTO]] = { dbMultipleResultCallWithAggregatedStatus( - getFlowCheckpointsFn( - GetFlowCheckpointsArgs(flowId, limit, offset, checkpointName, checkpointProperties, latestFirst, from, to) - ), + getFlowCheckpointsFn(GetFlowCheckpointsArgs(flowId, limit, offset, checkpointName, checkpointProperties)), "getFlowCheckpoints" ) .map(_.flatten) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointService.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointService.scala index c59ca56cd..4e6d4439e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointService.scala @@ -22,7 +22,6 @@ import za.co.absa.atum.server.model.PaginatedResult import zio.IO import zio.macros.accessible -import java.time.ZonedDateTime import java.util.UUID @accessible @@ -43,8 +42,6 @@ trait CheckpointService { checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], - from: Option[ZonedDateTime], - to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ServiceError, PaginatedResult[CheckpointV2DTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceImpl.scala index a53b57285..05428e894 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceImpl.scala @@ -23,7 +23,6 @@ import za.co.absa.atum.server.api.v2.repository.CheckpointRepository import za.co.absa.atum.server.model.PaginatedResult import zio._ -import java.time.ZonedDateTime import java.util.UUID class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends CheckpointService with BaseService { @@ -53,8 +52,6 @@ class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], - from: Option[ZonedDateTime], - to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ServiceError, PaginatedResult[CheckpointV2DTO]] = { repositoryCall( @@ -65,8 +62,6 @@ class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends checkpointName, checkpointProperties, latestFirst, - from, - to, includeProperties ), "getPartitioningCheckpoints" diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowService.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowService.scala index 5563086b9..27e0bbe5c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowService.scala @@ -22,8 +22,6 @@ import za.co.absa.atum.server.model.PaginatedResult import zio._ import zio.macros.accessible -import java.time.ZonedDateTime - @accessible trait FlowService { @@ -33,9 +31,6 @@ trait FlowService { offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], - latestFirst: Option[Boolean], - from: Option[ZonedDateTime], - to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ServiceError, PaginatedResult[CheckpointWithPartitioningDTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowServiceImpl.scala index 555b23271..936979a0c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowServiceImpl.scala @@ -23,8 +23,6 @@ import za.co.absa.atum.server.api.v2.repository.FlowRepository import za.co.absa.atum.server.model.PaginatedResult import zio._ -import java.time.ZonedDateTime - class FlowServiceImpl(flowRepository: FlowRepository) extends FlowService with BaseService { override def getFlowCheckpoints( @@ -33,23 +31,10 @@ class FlowServiceImpl(flowRepository: FlowRepository) extends FlowService with B offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], - latestFirst: Option[Boolean], - from: Option[ZonedDateTime], - to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ServiceError, PaginatedResult[CheckpointWithPartitioningDTO]] = { repositoryCall( - flowRepository.getFlowCheckpoints( - flowId, - limit, - offset, - checkpointName, - checkpointProperties, - latestFirst, - from, - to, - includeProperties - ), + flowRepository.getFlowCheckpoints(flowId, limit, offset, checkpointName, checkpointProperties, includeProperties), "getFlowCheckpoints" ) } diff --git a/server/src/main/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDB.scala index 2ad4d5337..25b1cf9e7 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDB.scala @@ -98,8 +98,10 @@ object CheckpointItemWithPartitioningFromDB { checkpointItems: Seq[CheckpointItemWithPartitioningFromDB] ): Either[Throwable, Seq[CheckpointWithPartitioningDTO]] = { val groupedItems = checkpointItems.groupBy(_.idCheckpoint) - // the DB function returns the items already ordered (latest or earliest first, as requested), so keep that order - val orderedCheckpointIds = checkpointItems.map(_.idCheckpoint).distinct + val orderedCheckpointIds = checkpointItems + .sortBy(_.checkpointStartTime)(Ordering[ZonedDateTime].reverse) + .map(_.idCheckpoint) + .distinct val result = orderedCheckpointIds.map { id => fromItemsToCheckpointWithPartitioningDTO(groupedItems(id)) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala index f2fc9a9a3..31147df05 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala @@ -25,8 +25,6 @@ import zio._ import zio.test._ import zio.interop.catz.asyncInstance -import java.time.ZonedDateTime - object GetFlowCheckpointsIntegrationTests extends ConfigProviderTest { override def spec: Spec[TestEnvironment with Scope, Any] = { @@ -39,10 +37,7 @@ object GetFlowCheckpointsIntegrationTests extends ConfigProviderTest { limit = 10, offset = 0L, checkpointName = Some("TestCheckpointName"), - checkpointProperties = None, - latestFirst = None, - from = None, - to = None + checkpointProperties = None ) for { @@ -56,27 +51,7 @@ object GetFlowCheckpointsIntegrationTests extends ConfigProviderTest { limit = 10, offset = 0L, checkpointName = Some("TestCheckpointName"), - checkpointProperties = Some(Map("key1" -> Seq("value1", "value2"))), - latestFirst = None, - from = None, - to = None - ) - - for { - getFlowCheckpoints <- ZIO.service[GetFlowCheckpoints] - result <- getFlowCheckpoints(args) - } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(42, "Flow not found")))) - }, - test("Should bind the latest-first flag and the process start time window") { - val args = GetFlowCheckpoints.GetFlowCheckpointsArgs( - flowId = 1L, - limit = 10, - offset = 0L, - checkpointName = None, - checkpointProperties = None, - latestFirst = Some(false), - from = Some(ZonedDateTime.parse("2026-06-01T00:00:00Z")), - to = Some(ZonedDateTime.parse("2026-08-01T00:00:00+02:00")) + checkpointProperties = Some(Map("key1" -> Seq("value1", "value2"))) ) for { diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala index f4e359fd1..3414b9cfb 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala @@ -26,8 +26,6 @@ import zio.interop.catz.asyncInstance import zio.{Scope, ZIO} import zio.test._ -import java.time.ZonedDateTime - object GetPartitioningCheckpointsIntegrationTests extends ConfigProviderTest { override def spec: Spec[TestEnvironment with Scope, Any] = { @@ -35,21 +33,13 @@ object GetPartitioningCheckpointsIntegrationTests extends ConfigProviderTest { test("Returns expected sequence of Checkpoints with non-existing partitioning id") { for { getPartitioningCheckpoints <- ZIO.service[GetPartitioningCheckpoints] - result <- getPartitioningCheckpoints(GetPartitioningCheckpointsArgs(0L, 10, 0L, None, None, None, None, None)) + result <- getPartitioningCheckpoints(GetPartitioningCheckpointsArgs(0L, 10, 0L, None, None, None)) } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) }, test("Should apply pagination (limit and offset) accurately with combined filters") { for { getPartitioningCheckpoints <- ZIO.service[GetPartitioningCheckpoints] - result <- getPartitioningCheckpoints(GetPartitioningCheckpointsArgs(0L, 10, 0L, Some("TestCheckpointName"), Some(Map("key1" -> Seq("value1", "value2"))), None, None, None)) - } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) - }, - test("Should bind the process start time window") { - val from = Some(ZonedDateTime.parse("2026-06-01T00:00:00Z")) - val to = Some(ZonedDateTime.parse("2026-08-01T00:00:00+02:00")) - for { - getPartitioningCheckpoints <- ZIO.service[GetPartitioningCheckpoints] - result <- getPartitioningCheckpoints(GetPartitioningCheckpointsArgs(0L, 10, 0L, None, None, Some(true), from, to)) + result <- getPartitioningCheckpoints(GetPartitioningCheckpointsArgs(0L, 10, 0L, Some("TestCheckpointName"), Some(Map("key1" -> Seq("value1", "value2"))), None)) } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) } ).provide( diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerUnitTests.scala index b44674f70..9a846d9d5 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerUnitTests.scala @@ -51,11 +51,11 @@ object CheckpointControllerUnitTests extends ConfigProviderTest with TestData { when(checkpointServiceMock.getCheckpoint(partitioningId1, checkpointV2DTO3.id, includeProperties = false)) .thenReturn(ZIO.fail(GeneralServiceError("boom!"))) - when(checkpointServiceMock.getPartitioningCheckpoints(partitioningId1, 10, 0L, None, None, None, None, None, includeProperties = false)) + when(checkpointServiceMock.getPartitioningCheckpoints(partitioningId1, 10, 0L, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultHasMore(Seq(checkpointV2DTO1)))) - when(checkpointServiceMock.getPartitioningCheckpoints(partitioningId2, 10, 0L, None, None, None, None, None, includeProperties = false)) + when(checkpointServiceMock.getPartitioningCheckpoints(partitioningId2, 10, 0L, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultNoMore(Seq(checkpointV2DTO1)))) - when(checkpointServiceMock.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, None, None, includeProperties = false)) + when(checkpointServiceMock.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundServiceError("Partitioning not found"))) private val checkpointServiceMockLayer = ZLayer.succeed(checkpointServiceMock) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerUnitTests.scala index eabe38774..d78be581e 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerUnitTests.scala @@ -29,11 +29,11 @@ import zio.test._ object FlowControllerUnitTests extends ZIOSpecDefault with TestData { private val flowServiceMock = mock(classOf[FlowService]) - when(flowServiceMock.getFlowCheckpoints(1L, 5, 2L, None, None, None, None, None, includeProperties = false)) + when(flowServiceMock.getFlowCheckpoints(1L, 5, 2L, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultHasMore(Seq(checkpointWithPartitioningDTO1)))) - when(flowServiceMock.getFlowCheckpoints(2L, 5, 0L, None, None, None, None, None, includeProperties = false)) + when(flowServiceMock.getFlowCheckpoints(2L, 5, 0L, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultNoMore(Seq(checkpointWithPartitioningDTO2)))) - when(flowServiceMock.getFlowCheckpoints(3L, 5, 0L, None, None, None, None, None, includeProperties = false)) + when(flowServiceMock.getFlowCheckpoints(3L, 5, 0L, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundServiceError("Flow not found"))) private val flowServiceMockLayer = ZLayer.succeed(flowServiceMock) @@ -43,20 +43,20 @@ object FlowControllerUnitTests extends ZIOSpecDefault with TestData { suite("GetFlowCheckpointsV2Suite")( test("Returns expected Seq[CheckpointV2DTO] with Pagination indicating there is more data available") { for { - result <- FlowController.getFlowCheckpoints(1L, 5, 2L, None, None, None, None, None, includeProperties = false) + result <- FlowController.getFlowCheckpoints(1L, 5, 2L, None, None, includeProperties = false) } yield assertTrue( result.data == Seq(checkpointWithPartitioningDTO1) && result.pagination == Pagination(5, 2, hasMore = true) ) }, test("Returns expected Seq[CheckpointV2DTO] with Pagination indicating there is no more data available") { for { - result <- FlowController.getFlowCheckpoints(2L, 5, 0L, None, None, None, None, None, includeProperties = false) + result <- FlowController.getFlowCheckpoints(2L, 5, 0L, None, None, includeProperties = false) } yield assertTrue( result.data == Seq(checkpointWithPartitioningDTO2) && result.pagination == Pagination(5, 0, hasMore = false) ) }, test("Returns expected NotFoundServiceError when service returns NotFoundServiceError") { - assertZIO(FlowController.getFlowCheckpoints(3L, 5, 0L, None, None, None, None, None, includeProperties = false).exit)( + assertZIO(FlowController.getFlowCheckpoints(3L, 5, 0L, None, None, includeProperties = false).exit)( failsWithA[NotFoundErrorResponse] ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala index b9f72acf7..be500cc6e 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala @@ -32,7 +32,6 @@ import zio.test.Assertion.equalTo import zio.test.{Spec, TestEnvironment, ZIOSpecDefault, assertZIO} import zio.{Scope, ZIO, ZLayer} -import java.time.ZonedDateTime import java.util.UUID object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData { @@ -46,11 +45,11 @@ object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData private val encodedExecutionIdProperties = "eyJleGVjdXRpb25JRCI6IjAxOWY4OTgxLTc4NjgtNzlmYy04MWQzLTgxNDNhNDcwNmY4YSJ9" - when(flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, None, None, None, None, includeProperties = false)) + when(flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, None, includeProperties = false)) .thenReturn( ZIO.succeed(PaginatedResponse(Seq(checkpointWithPartitioningDTO1), Pagination(5, 0, hasMore = true), uuid)) ) - when(flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, None, None, None, None, includeProperties = true)) + when(flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, None, includeProperties = true)) .thenReturn( ZIO.succeed( PaginatedResponse( @@ -60,56 +59,24 @@ object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData ) ) ) - when(flowControllerMockV2.getFlowCheckpoints(2L, 5, 0L, None, None, None, None, None, includeProperties = false)) + when(flowControllerMockV2.getFlowCheckpoints(2L, 5, 0L, None, None, includeProperties = false)) .thenReturn( ZIO.succeed(PaginatedResponse(Seq(checkpointWithPartitioningDTO2), Pagination(5, 0, hasMore = false), uuid)) ) - when(flowControllerMockV2.getFlowCheckpoints(3L, 5, 0L, None, None, None, None, None, includeProperties = false)) + when(flowControllerMockV2.getFlowCheckpoints(3L, 5, 0L, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundErrorResponse("Flow not found for a given ID"))) when( - flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, Some(executionIdProperties.view.mapValues(Seq(_)).toMap), None, None, None, includeProperties = false) + flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, Some(executionIdProperties.view.mapValues(Seq(_)).toMap), includeProperties = false) ) .thenReturn( ZIO.succeed(PaginatedResponse(Seq(checkpointWithPartitioningDTO1), Pagination(5, 0, hasMore = true), uuid)) ) - private val windowFrom = ZonedDateTime.parse("2026-06-01T00:00:00Z") - private val windowTo = ZonedDateTime.parse("2026-08-01T00:00:00Z") - - when( - flowControllerMockV2.getFlowCheckpoints( - 1L, 5, 0L, None, None, Some(false), Some(windowFrom), Some(windowTo), includeProperties = false - ) - ) - .thenReturn( - ZIO.succeed(PaginatedResponse(Seq(checkpointWithPartitioningDTO2), Pagination(5, 0, hasMore = false), uuid)) - ) - private val flowControllerMockLayerV2 = ZLayer.succeed(flowControllerMockV2) private val getFlowCheckpointServerEndpoint = Endpoints.getFlowCheckpointsEndpoint.zServerLogic({ - case ( - flowId: Long, - limit: Int, - offset: Long, - checkpointName: Option[String], - checkpointProperties: Option[Map[String, Seq[String]]], - latestFirst: Option[Boolean], - from: Option[ZonedDateTime], - to: Option[ZonedDateTime], - includeProperties: Boolean - ) => - FlowController.getFlowCheckpoints( - flowId, - limit, - offset, - checkpointName, - checkpointProperties, - latestFirst, - from, - to, - includeProperties - ) + case (flowId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], includeProperties: Boolean) => + FlowController.getFlowCheckpoints(flowId, limit, offset, checkpointName, checkpointProperties, includeProperties) }) def spec: Spec[TestEnvironment with Scope, Any] = { @@ -207,47 +174,6 @@ object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData ) ) }, - test("Returns an expected PaginatedResponse[CheckpointWithPartitioningDTO] when filtering by a time window, earliest first") { - val baseUri = - uri"https://test.com/api/v2/flows/1/checkpoints?limit=5&offset=0&latest-first=false&from=2026-06-01T00:00:00Z&to=2026-08-01T00:00:00Z" - val response = basicRequest - .get(baseUri) - .response(asJson[PaginatedResponse[CheckpointWithPartitioningDTO]]) - .send(backendStub) - - val body = response.map(_.body) - val statusCode = response.map(_.code) - - assertZIO(body <&> statusCode)( - equalTo( - Right(PaginatedResponse(Seq(checkpointWithPartitioningDTO2), Pagination(5, 0, hasMore = false), uuid)), - StatusCode.Ok - ) - ) - }, - test("Returns expected 400 when 'to' is not a valid date-time") { - val baseUri = uri"https://test.com/api/v2/flows/1/checkpoints?limit=5&offset=0&to=not-a-date-time" - val response = basicRequest - .get(baseUri) - .response(asJson[PaginatedResponse[CheckpointWithPartitioningDTO]]) - .send(backendStub) - - val statusCode = response.map(_.code) - - assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) - }, - test("Returns expected 400 when 'from' is after 'to'") { - val baseUri = - uri"https://test.com/api/v2/flows/1/checkpoints?limit=5&offset=0&from=2026-08-01T00:00:00Z&to=2026-06-01T00:00:00Z" - val response = basicRequest - .get(baseUri) - .response(asJson[PaginatedResponse[CheckpointWithPartitioningDTO]]) - .send(backendStub) - - val statusCode = response.map(_.code) - - assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) - }, test("Returns expected 400 when checkpoint-properties is not valid base64") { val baseUri = uri"https://test.com/api/v2/flows/1/checkpoints?limit=5&offset=0&checkpoint-properties=!!!not-base64!!!" diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala index 3e634a284..9f5e0d4ad 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala @@ -32,7 +32,6 @@ import zio.test.Assertion.equalTo import zio.test.{Spec, TestEnvironment, ZIOSpecDefault, assertZIO} import zio.{Scope, ZIO, ZLayer} -import java.time.ZonedDateTime import java.util.UUID object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData { @@ -41,12 +40,9 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T private val uuid = UUID.randomUUID() - private val windowFrom = ZonedDateTime.parse("2026-06-01T00:00:00Z") - private val windowTo = ZonedDateTime.parse("2026-08-01T00:00:00Z") - - when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, None, None, includeProperties = false)) + when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(PaginatedResponse(Seq(checkpointV2DTO1), Pagination(10, 0, hasMore = true), uuid))) - when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, None, None, includeProperties = true)) + when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, includeProperties = true)) .thenReturn( ZIO.succeed( PaginatedResponse( @@ -56,20 +52,14 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T ) ) ) - when(checkpointControllerMock.getPartitioningCheckpoints(1L, 20, 0L, None, None, None, None, None, includeProperties = false)) + when(checkpointControllerMock.getPartitioningCheckpoints(1L, 20, 0L, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(PaginatedResponse(Seq(checkpointV2DTO1), Pagination(20, 0, hasMore = false), uuid))) - when(checkpointControllerMock.getPartitioningCheckpoints(2L, 10, 0L, None, None, None, None, None, includeProperties = false)) + when(checkpointControllerMock.getPartitioningCheckpoints(2L, 10, 0L, None, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundErrorResponse("partitioning not found"))) - when(checkpointControllerMock.getPartitioningCheckpoints(3L, 10, 0L, None, None, None, None, None, includeProperties = false)) + when(checkpointControllerMock.getPartitioningCheckpoints(3L, 10, 0L, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(PaginatedResponse(Seq(checkpointV2DTO1), Pagination(10, 0, hasMore = true), uuid))) - when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, Some(Map("executionID" -> Seq("id1", "id2"))), None, None, None, includeProperties = false)) + when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, Some(Map("executionID" -> Seq("id1", "id2"))), None, includeProperties = false)) .thenReturn(ZIO.succeed(PaginatedResponse(Seq(checkpointV2DTO1), Pagination(10, 0, hasMore = true), uuid))) - when( - checkpointControllerMock.getPartitioningCheckpoints( - 1L, 10, 0L, None, None, None, Some(windowFrom), Some(windowTo), includeProperties = false - ) - ) - .thenReturn(ZIO.succeed(PaginatedResponse(Seq(checkpointV2DTO2), Pagination(10, 0, hasMore = false), uuid))) private val checkpointControllerMockLayer = ZLayer.succeed(checkpointControllerMock) @@ -83,8 +73,6 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], - from: Option[ZonedDateTime], - to: Option[ZonedDateTime], includeProperties: Boolean ) => CheckpointController.getPartitioningCheckpoints( @@ -94,8 +82,6 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T checkpointName, checkpointProperties, latestFirst, - from, - to, includeProperties ) }) @@ -129,52 +115,6 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T ) ) }, - test("Returns an expected PaginatedResponse[CheckpointV2DTO] when filtering by a process start time window") { - val request = basicRequest - .get( - uri"https://test.com/api/v2/partitionings/1/checkpoints?limit=10&offset=0&from=2026-06-01T00:00:00Z&to=2026-08-01T00:00:00Z" - ) - .response(asJson[PaginatedResponse[CheckpointV2DTO]]) - - val response = request - .send(backendStub) - - val body = response.map(_.body) - val statusCode = response.map(_.code) - - assertZIO(body <&> statusCode)( - equalTo( - Right(PaginatedResponse(Seq(checkpointV2DTO2), Pagination(10, 0, hasMore = false), uuid)), - StatusCode.Ok - ) - ) - }, - test("Returns expected 400 when 'from' is not a valid date-time") { - val request = basicRequest - .get(uri"https://test.com/api/v2/partitionings/1/checkpoints?limit=10&offset=0&from=2026-06-01") - .response(asJson[PaginatedResponse[CheckpointV2DTO]]) - - val response = request - .send(backendStub) - - val statusCode = response.map(_.code) - - assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) - }, - test("Returns expected 400 when 'from' is not before 'to'") { - val request = basicRequest - .get( - uri"https://test.com/api/v2/partitionings/1/checkpoints?limit=10&offset=0&from=2026-08-01T00:00:00Z&to=2026-08-01T00:00:00Z" - ) - .response(asJson[PaginatedResponse[CheckpointV2DTO]]) - - val response = request - .send(backendStub) - - val statusCode = response.map(_.code) - - assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) - }, test("Returns an expected PaginatedResponse[CheckpointV2DTO] with more data available") { val request = basicRequest .get(uri"https://test.com/api/v2/partitionings/1/checkpoints?limit=10&offset=0") diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryUnitTests.scala index ef54fb94e..af5308870 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryUnitTests.scala @@ -62,11 +62,11 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { when(getCheckpointMockV2.apply(GetPartitioningCheckpointV2Args(partitioningId, checkpointV2DTO3.id))) .thenReturn(ZIO.fail(new Exception("boom!"))) - when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(0L, 10, 0L, None, None, None, None, None))) + when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(0L, 10, 0L, None, None, None))) .thenReturn(ZIO.right(Seq(Row(FunctionStatus(11, "ok"), Some(checkpointItemPaginatedFromDB))))) - when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(1L, 10, 0L, None, None, None, None, None))) + when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(1L, 10, 0L, None, None, None))) .thenReturn(ZIO.right(Seq(Row(FunctionStatus(11, "ok"), None)))) - when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(3L, 10, 0L, None, None, None, None, None))) + when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(3L, 10, 0L, None, None, None))) .thenReturn(ZIO.left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) when(getCheckpointPropertiesMock.apply(GetCheckpointPropertiesArgs(checkpointV2DTO1.id))) @@ -137,14 +137,14 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { suite("GetPartitioningCheckpointsSuite")( test("Returns expected Seq without properties") { for { - result <- CheckpointRepository.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, None, None, includeProperties = false) + result <- CheckpointRepository.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, includeProperties = false) } yield assertTrue( result.isInstanceOf[ResultHasMore[CheckpointV2DTO]] && result.data == Seq(checkpointV2DTO2) ) }, test("Returns expected Seq with properties") { for { - result <- CheckpointRepository.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, None, None, includeProperties = true) + result <- CheckpointRepository.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, includeProperties = true) } yield assertTrue( result.isInstanceOf[ResultHasMore[CheckpointV2DTO]] && result.data == Seq( checkpointV2DTO2.copy(properties = Some(Map("propName1" -> "propValue1"))) @@ -153,13 +153,13 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { }, test("Returns expected Seq.empty") { for { - result <- CheckpointRepository.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, None, None, includeProperties = false) + result <- CheckpointRepository.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, includeProperties = false) } yield assertTrue( result.isInstanceOf[ResultNoMore[CheckpointV2DTO]] && result.data == Seq.empty[CheckpointV2DTO] ) }, test("Returns expected NotFoundDatabaseError") { - assertZIO(CheckpointRepository.getPartitioningCheckpoints(3L, 10, 0L, None, None, None, None, None, includeProperties = false).exit)( + assertZIO(CheckpointRepository.getPartitioningCheckpoints(3L, 10, 0L, None, None, None, includeProperties = false).exit)( failsWithA[NotFoundDatabaseError] ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryUnitTests.scala index 12c594385..ff30f4b86 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryUnitTests.scala @@ -38,7 +38,7 @@ object FlowRepositoryUnitTests extends ZIOSpecDefault with TestData { private val getFlowCheckpointsV2Mock = mock(classOf[GetFlowCheckpoints]) private val getCheckpointPropertiesMock: GetCheckpointProperties = mock(classOf[GetCheckpointProperties]) - when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(1, 1, 1L, None, None, None, None, None))) + when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(1, 1, 1L, None, None))) .thenReturn( ZIO.right( Seq( @@ -46,9 +46,9 @@ object FlowRepositoryUnitTests extends ZIOSpecDefault with TestData { ) ) ) - when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(2, 1, 1L, None, None, None, None, None))) + when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(2, 1, 1L, None, None))) .thenReturn(ZIO.right(Seq(Row(FunctionStatus(11, "success"), None)))) - when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(3, 1, 1L, None, None, None, None, None))) + when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(3, 1, 1L, None, None))) .thenReturn(ZIO.fail(DataNotFoundException(FunctionStatus(42, "Flow not found")))) when( @@ -67,12 +67,12 @@ object FlowRepositoryUnitTests extends ZIOSpecDefault with TestData { suite("GetFlowCheckpointsV2Suite")( test("Returns expected Right with CheckpointV2DTO without properties") { for { - result <- FlowRepository.getFlowCheckpoints(1, 1, 1L, None, None, None, None, None, includeProperties = false) + result <- FlowRepository.getFlowCheckpoints(1, 1, 1L, None, None, includeProperties = false) } yield assertTrue(result == ResultHasMore(Seq(checkpointWithPartitioningDTO1))) }, test("Returns expected Right with CheckpointV2DTO with properties") { for { - result <- FlowRepository.getFlowCheckpoints(1, 1, 1L, None, None, None, None, None, includeProperties = true) + result <- FlowRepository.getFlowCheckpoints(1, 1, 1L, None, None, includeProperties = true) } yield assertTrue( result == ResultHasMore( Seq(checkpointWithPartitioningDTO1.copy(properties = Some(Map("propName1" -> "propValue1")))) @@ -81,11 +81,11 @@ object FlowRepositoryUnitTests extends ZIOSpecDefault with TestData { }, test("Returns expected Right with CheckpointV2DTO") { for { - result <- FlowRepository.getFlowCheckpoints(2, 1, 1L, None, None, None, None, None, includeProperties = false) + result <- FlowRepository.getFlowCheckpoints(2, 1, 1L, None, None, includeProperties = false) } yield assertTrue(result == ResultNoMore(Seq.empty[CheckpointWithPartitioningDTO])) }, test("Returns expected DatabaseError") { - assertZIO(FlowRepository.getFlowCheckpoints(3, 1, 1L, None, None, None, None, None, includeProperties = false).exit)( + assertZIO(FlowRepository.getFlowCheckpoints(3, 1, 1L, None, None, includeProperties = false).exit)( failsWithA[NotFoundDatabaseError] ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceUnitTests.scala index eaba19979..e2c5406dc 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceUnitTests.scala @@ -45,9 +45,9 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { when(checkpointRepositoryMock.getCheckpoint(partitioningId, checkpointV2DTO2.id, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundDatabaseError("not found"))) - when(checkpointRepositoryMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, None, None, includeProperties = false)) + when(checkpointRepositoryMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultHasMore(Seq(checkpointV2DTO1)))) - when(checkpointRepositoryMock.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, None, None, includeProperties = false)) + when(checkpointRepositoryMock.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundDatabaseError("Partitioning not found"))) private val checkpointRepositoryMockLayer = ZLayer.succeed(checkpointRepositoryMock) @@ -92,13 +92,13 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { suite("GetPartitioningCheckpointsSuite")( test("Returns expected Right with Seq[CheckpointDTO]") { for { - result <- CheckpointService.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, None, None, includeProperties = false) + result <- CheckpointService.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, includeProperties = false) } yield assertTrue { result == ResultHasMore(Seq(checkpointV2DTO1)) } }, test("Returns expected NotFoundServiceError") { - assertZIO(CheckpointService.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, None, None, includeProperties = false).exit)( + assertZIO(CheckpointService.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, includeProperties = false).exit)( failsWithA[NotFoundServiceError] ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/service/FlowServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/service/FlowServiceUnitTests.scala index 51233afbe..bcd867305 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/service/FlowServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/service/FlowServiceUnitTests.scala @@ -29,9 +29,9 @@ import zio.test._ object FlowServiceUnitTests extends ZIOSpecDefault with TestData { private val flowRepositoryMock = mock(classOf[FlowRepository]) - when(flowRepositoryMock.getFlowCheckpoints(1L, 1, 1L, None, None, None, None, None, includeProperties = false)) + when(flowRepositoryMock.getFlowCheckpoints(1L, 1, 1L, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultHasMore(Seq(checkpointWithPartitioningDTO1)))) - when(flowRepositoryMock.getFlowCheckpoints(2L, 1, 1L, None, None, None, None, None, includeProperties = false)) + when(flowRepositoryMock.getFlowCheckpoints(2L, 1, 1L, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundDatabaseError("Flow not found"))) private val flowRepositoryMockLayer = ZLayer.succeed(flowRepositoryMock) @@ -42,13 +42,13 @@ object FlowServiceUnitTests extends ZIOSpecDefault with TestData { suite("GetFlowCheckpointsV2Suite")( test("Returns expected PaginatedResult[CheckpointV2DTO]") { for { - result <- FlowService.getFlowCheckpoints(1L, 1, 1L, None, None, None, None, None, includeProperties = false) + result <- FlowService.getFlowCheckpoints(1L, 1, 1L, None, None, includeProperties = false) } yield assertTrue { result == ResultHasMore(Seq(checkpointWithPartitioningDTO1)) } }, test("Returns expected ServiceError") { - assertZIO(FlowService.getFlowCheckpoints(2L, 1, 1L, None, None, None, None, None, includeProperties = false).exit)( + assertZIO(FlowService.getFlowCheckpoints(2L, 1, 1L, None, None, includeProperties = false).exit)( failsWithA[NotFoundServiceError] ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDBUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDBUnitTests.scala index 1be1b3625..87d2b75c6 100644 --- a/server/src/test/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDBUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDBUnitTests.scala @@ -24,7 +24,7 @@ import java.util.UUID class CheckpointItemWithPartitioningFromDBUnitTests extends AnyFunSuiteLike with TestData { - test("groupAndConvertItemsToCheckpointWithPartitioningDTOs should group by idCheckpoint and keep the order of the DB rows") { + test("groupAndConvertItemsToCheckpointWithPartitioningDTOs should group by idCheckpoint and sort by checkpointStartTime in descending order") { val checkpointItem1 = CheckpointItemWithPartitioningFromDB( idCheckpoint = UUID.randomUUID(), checkpointName = "checkpoint1", @@ -57,20 +57,14 @@ class CheckpointItemWithPartitioningFromDBUnitTests extends AnyFunSuiteLike with hasMore = true ) - // a second measure of the first checkpoint, i.e. another DB row of the same checkpoint - val checkpointItem1OtherMeasure = checkpointItem1.copy(measureName = "measure3", measuredColumns = Seq("col3")) + val checkpointItems = Seq(checkpointItem1, checkpointItem2) - // earliest first, as returned by the DB for latest-first = false - val earliestFirst = CheckpointItemWithPartitioningFromDB - .groupAndConvertItemsToCheckpointWithPartitioningDTOs(Seq(checkpointItem1, checkpointItem1OtherMeasure, checkpointItem2)) - assert(earliestFirst.isRight) - assert(earliestFirst.toOption.get.map(_.id) == Seq(checkpointItem1.idCheckpoint, checkpointItem2.idCheckpoint)) - assert(earliestFirst.toOption.get.head.measurements.size == 2) + val result = CheckpointItemWithPartitioningFromDB.groupAndConvertItemsToCheckpointWithPartitioningDTOs(checkpointItems) - // latest first, as returned by the DB by default - val latestFirst = CheckpointItemWithPartitioningFromDB - .groupAndConvertItemsToCheckpointWithPartitioningDTOs(Seq(checkpointItem2, checkpointItem1, checkpointItem1OtherMeasure)) - assert(latestFirst.isRight) - assert(latestFirst.toOption.get.map(_.id) == Seq(checkpointItem2.idCheckpoint, checkpointItem1.idCheckpoint)) + assert(result.isRight) + val checkpoints = result.toOption.get + + assert(checkpoints.size == 2) + assert(checkpoints.head.processStartTime.isAfter(checkpoints(1).processStartTime)) } } From 5467d7d40acae008c9017c9a4f92077bd55d65ef Mon Sep 17 00:00:00 2001 From: Ladislav Sulak Date: Fri, 9 Oct 2026 17:36:10 +0200 Subject: [PATCH 10/21] feat(db): keep final checkpoint function signatures and properties index in #478 Ship the final shape of flows.get_flow_checkpoints and runs.get_partitioning_checkpoints (JSONB multi-value properties filter plus optional i_from_time / i_to_time, both DEFAULT NULL) in this release, so the follow-up audit PR does not need another full copy of both functions. Positional callers are unaffected: the server keeps passing the leading arguments only. Also keep the (fk_checkpoint, property_name) index on runs.checkpoint_properties, now V0.8.0.3: the properties filter looks checkpoints up by fk_checkpoint, which had no index. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../flows/V0.8.0.1__get_flow_checkpoints.sql | 20 ++- ...V0.8.0.2__get_partitioning_checkpoints.sql | 20 ++- ...checkpoint_properties_checkpoint_index.ddl | 23 +++ .../GetFlowCheckpointsIntegrationTests.scala | 144 +++++++++++++++++- ...titioningCheckpointsIntegrationTests.scala | 116 +++++++++++++- 5 files changed, 309 insertions(+), 14 deletions(-) create mode 100644 database/src/main/postgres/runs/V0.8.0.3__checkpoint_properties_checkpoint_index.ddl diff --git a/database/src/main/postgres/flows/V0.8.0.1__get_flow_checkpoints.sql b/database/src/main/postgres/flows/V0.8.0.1__get_flow_checkpoints.sql index d89c828b6..74c90e200 100644 --- a/database/src/main/postgres/flows/V0.8.0.1__get_flow_checkpoints.sql +++ b/database/src/main/postgres/flows/V0.8.0.1__get_flow_checkpoints.sql @@ -14,8 +14,8 @@ * limitations under the License. */ --- The previous overload is dropped so that the new signature (JSONB multi-value checkpoint properties filter) is --- unambiguous when the function is called with named or defaulted arguments. +-- The previous overload is dropped so that the new signature (JSONB multi-value checkpoint properties filter and +-- the process start time window) is unambiguous when the function is called with named or defaulted arguments. DROP FUNCTION IF EXISTS flows.get_flow_checkpoints(BIGINT, INT, BIGINT, TEXT, HSTORE, BOOLEAN); CREATE OR REPLACE FUNCTION flows.get_flow_checkpoints( @@ -25,6 +25,8 @@ CREATE OR REPLACE FUNCTION flows.get_flow_checkpoints( IN i_checkpoint_name TEXT DEFAULT NULL, IN i_checkpoint_properties JSONB DEFAULT NULL, IN i_latest_first BOOLEAN DEFAULT TRUE, + IN i_from_time TIMESTAMP WITH TIME ZONE DEFAULT NULL, + IN i_to_time TIMESTAMP WITH TIME ZONE DEFAULT NULL, OUT status INTEGER, OUT status_text TEXT, OUT id_checkpoint UUID, @@ -43,9 +45,9 @@ CREATE OR REPLACE FUNCTION flows.get_flow_checkpoints( ) RETURNS SETOF record AS -------------------------------------------------------------------------------------------------------------------- -- --- Function: flows.get_flow_checkpoints(6) +-- Function: flows.get_flow_checkpoints(8) -- Retrieves all checkpoints (measures and their measurement details) related to an input flow (and checkpoint --- name and/or checkpoint properties, if specified). +-- name, checkpoint properties and/or process start time window, if specified). -- -- Note: a single row returned from this function doesn't contain all data related to a single checkpoint - it only -- represents one measure associated with a checkpoint. So even if only a single checkpoint would be retrieved, @@ -61,6 +63,8 @@ CREATE OR REPLACE FUNCTION flows.get_flow_checkpoints( -- property name, have that property with one of the accepted values -- i_latest_first - (optional) if true (default), checkpoints are ordered by process_start_time -- in descending order (latest first); if false, in ascending order +-- i_from_time - (optional) if specified, returns only checkpoints with process_start_time >= i_from_time +-- i_to_time - (optional) if specified, returns only checkpoints with process_start_time < i_to_time -- -- Note: i_checkpoints_limit and i_offset are used for pagination purposes; -- checkpoints are ordered by process_start_time (descending by default, see i_latest_first) @@ -115,6 +119,8 @@ BEGIN JOIN flows.partitioning_to_flow PF ON C.fk_partitioning = PF.fk_partitioning WHERE PF.fk_flow = i_flow_id AND (i_checkpoint_name IS NULL OR C.checkpoint_name = i_checkpoint_name) + AND (i_from_time IS NULL OR C.process_start_time >= i_from_time) + AND (i_to_time IS NULL OR C.process_start_time < i_to_time) AND ( i_checkpoint_properties IS NULL OR NOT EXISTS ( @@ -155,6 +161,8 @@ BEGIN JOIN flows.partitioning_to_flow PF ON C.fk_partitioning = PF.fk_partitioning WHERE PF.fk_flow = i_flow_id AND (i_checkpoint_name IS NULL OR C.checkpoint_name = i_checkpoint_name) + AND (i_from_time IS NULL OR C.process_start_time >= i_from_time) + AND (i_to_time IS NULL OR C.process_start_time < i_to_time) AND ( i_checkpoint_properties IS NULL OR NOT EXISTS ( @@ -209,5 +217,5 @@ BEGIN END; $$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER; -ALTER FUNCTION flows.get_flow_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN) OWNER TO atum_owner; -GRANT EXECUTE ON FUNCTION flows.get_flow_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN) TO atum_owner; +ALTER FUNCTION flows.get_flow_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE) OWNER TO atum_owner; +GRANT EXECUTE ON FUNCTION flows.get_flow_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE) TO atum_owner; diff --git a/database/src/main/postgres/runs/V0.8.0.2__get_partitioning_checkpoints.sql b/database/src/main/postgres/runs/V0.8.0.2__get_partitioning_checkpoints.sql index 212f5fa28..76ad85905 100644 --- a/database/src/main/postgres/runs/V0.8.0.2__get_partitioning_checkpoints.sql +++ b/database/src/main/postgres/runs/V0.8.0.2__get_partitioning_checkpoints.sql @@ -14,8 +14,8 @@ * limitations under the License. */ --- The previous overload is dropped so that the new signature (JSONB multi-value checkpoint properties filter) is --- unambiguous when the function is called with named or defaulted arguments. +-- The previous overload is dropped so that the new signature (JSONB multi-value checkpoint properties filter and +-- the process start time window) is unambiguous when the function is called with named or defaulted arguments. DROP FUNCTION IF EXISTS runs.get_partitioning_checkpoints(BIGINT, INT, BIGINT, TEXT, HSTORE, BOOLEAN); CREATE OR REPLACE FUNCTION runs.get_partitioning_checkpoints( @@ -25,6 +25,8 @@ CREATE OR REPLACE FUNCTION runs.get_partitioning_checkpoints( IN i_checkpoint_name TEXT DEFAULT NULL, IN i_checkpoint_properties JSONB DEFAULT NULL, IN i_latest_first BOOLEAN DEFAULT TRUE, + IN i_from_time TIMESTAMP WITH TIME ZONE DEFAULT NULL, + IN i_to_time TIMESTAMP WITH TIME ZONE DEFAULT NULL, OUT status INTEGER, OUT status_text TEXT, OUT id_checkpoint UUID, @@ -40,9 +42,9 @@ CREATE OR REPLACE FUNCTION runs.get_partitioning_checkpoints( ) RETURNS SETOF record AS -------------------------------------------------------------------------------------------------------------------- -- --- Function: runs.get_partitioning_checkpoints(6) +-- Function: runs.get_partitioning_checkpoints(8) -- Retrieves all checkpoints (measures and their measurement details) related to an input partitioning (and --- checkpoint name and/or checkpoint properties, if specified). +-- checkpoint name, checkpoint properties and/or process start time window, if specified). -- -- Note: a single row returned from this function doesn't contain all data related to a single checkpoint - it only -- represents one measure associated with a checkpoint. So even if only a single checkpoint would be retrieved, @@ -58,6 +60,8 @@ CREATE OR REPLACE FUNCTION runs.get_partitioning_checkpoints( -- property name, have that property with one of the accepted values -- i_latest_first - (optional) if true (default), checkpoints are ordered by process_start_time -- in descending order (latest first); if false, in ascending order +-- i_from_time - (optional) if specified, returns only checkpoints with process_start_time >= i_from_time +-- i_to_time - (optional) if specified, returns only checkpoints with process_start_time < i_to_time -- -- Note: i_checkpoints_limit and i_offset are used for pagination purposes; -- checkpoints are ordered by process_start_time (descending by default, see i_latest_first) @@ -102,6 +106,8 @@ BEGIN FROM runs.checkpoints C WHERE C.fk_partitioning = i_partitioning_id AND (i_checkpoint_name IS NULL OR C.checkpoint_name = i_checkpoint_name) + AND (i_from_time IS NULL OR C.process_start_time >= i_from_time) + AND (i_to_time IS NULL OR C.process_start_time < i_to_time) AND ( i_checkpoint_properties IS NULL OR NOT EXISTS ( @@ -140,6 +146,8 @@ BEGIN FROM runs.checkpoints C WHERE C.fk_partitioning = i_partitioning_id AND (i_checkpoint_name IS NULL OR C.checkpoint_name = i_checkpoint_name) + AND (i_from_time IS NULL OR C.process_start_time >= i_from_time) + AND (i_to_time IS NULL OR C.process_start_time < i_to_time) AND ( i_checkpoint_properties IS NULL OR NOT EXISTS ( @@ -189,5 +197,5 @@ BEGIN END; $$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER; -ALTER FUNCTION runs.get_partitioning_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN) OWNER TO atum_owner; -GRANT EXECUTE ON FUNCTION runs.get_partitioning_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN) TO atum_owner; +ALTER FUNCTION runs.get_partitioning_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE) OWNER TO atum_owner; +GRANT EXECUTE ON FUNCTION runs.get_partitioning_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE) TO atum_owner; diff --git a/database/src/main/postgres/runs/V0.8.0.3__checkpoint_properties_checkpoint_index.ddl b/database/src/main/postgres/runs/V0.8.0.3__checkpoint_properties_checkpoint_index.ddl new file mode 100644 index 000000000..bf7f54a0f --- /dev/null +++ b/database/src/main/postgres/runs/V0.8.0.3__checkpoint_properties_checkpoint_index.ddl @@ -0,0 +1,23 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Supports the checkpoint properties filter of runs.get_partitioning_checkpoints and flows.get_flow_checkpoints +-- (looked up per checkpoint and property name), and the retrieval of a checkpoint's properties. +-- +-- Built CONCURRENTLY so writes to runs.checkpoint_properties are not blocked. This file must contain only this +-- statement: Flyway then runs it outside of a transaction, which CONCURRENTLY requires. +-- If the build fails, it leaves an INVALID index behind; drop it +-- (DROP INDEX CONCURRENTLY runs.idx_checkpoint_properties_checkpoint_name) and re-run the migration. +CREATE INDEX CONCURRENTLY idx_checkpoint_properties_checkpoint_name ON runs.checkpoint_properties (fk_checkpoint, property_name); diff --git a/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala index 62ad5e6f4..f3c71a747 100644 --- a/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala @@ -3,7 +3,7 @@ package za.co.absa.atum.database.flows import io.circe.Json import io.circe.parser.parse import za.co.absa.balta.DBTestSuite -import za.co.absa.balta.classes.JsonBString +import za.co.absa.balta.classes.{JsonBString, QueryResult} import za.co.absa.balta.classes.setter.CustomDBType import java.time.OffsetDateTime @@ -847,6 +847,148 @@ class GetFlowCheckpointsIntegrationTests extends DBTestSuite { } } + test("getFlowCheckpointsV2 should return only checkpoints within the process start time window, across partitionings") { + + val otherPartitioning = JsonBString( + """ + |{ + | "version": 1, + | "keys": ["keyX", "keyY", "keyZ"], + | "keysToValuesMap": { + | "keyX": "value1", + | "keyZ": "value3", + | "keyY": "otherValue2" + | } + |} + |""".stripMargin + ) + + val partitioningIdA: Long = Random.nextLong() + val partitioningIdB: Long = Random.nextLong() + table("runs.partitionings").insert( + add("id_partitioning", partitioningIdA) + .add("partitioning", partitioning) + .add("created_by", "Joseph") + ) + table("runs.partitionings").insert( + add("id_partitioning", partitioningIdB) + .add("partitioning", otherPartitioning) + .add("created_by", "Joseph") + ) + + val flowId: Long = Random.nextLong() + table("flows.flows").insert( + add("id_flow", flowId) + .add("flow_name", "flowNameTimeWindow") + .add("from_pattern", false) + .add("created_by", "Joseph") + .add("fk_primary_partitioning", partitioningIdA) + ) + Seq(partitioningIdA, partitioningIdB).foreach { partitioningId => + table("flows.partitioning_to_flow").insert( + add("fk_flow", flowId) + .add("fk_partitioning", partitioningId) + .add("created_by", "ObviouslySomeTest") + ) + } + + val measureDefinitionIdA: Long = Random.nextLong() + val measureDefinitionIdB: Long = Random.nextLong() + Seq(partitioningIdA -> measureDefinitionIdA, partitioningIdB -> measureDefinitionIdB).foreach { + case (partitioningId, measureDefinitionId) => + table("runs.measure_definitions").insert( + add("id_measure_definition", measureDefinitionId) + .add("fk_partitioning", partitioningId) + .add("measure_name", "cnt") + .add("measured_columns", CustomDBType("""{"col1"}""", "TEXT[]")) + .add("created_by", "Joseph") + ) + } + + // one checkpoint at the start of each month, alternating between the two partitionings of the flow + case class TestCheckpoint(id: UUID, partitioningId: Long, measureDefinitionId: Long, start: OffsetDateTime, executionId: String) + val jan = TestCheckpoint(UUID.randomUUID(), partitioningIdA, measureDefinitionIdA, OffsetDateTime.parse("2026-01-01T00:00:00Z"), "e1") + val feb = TestCheckpoint(UUID.randomUUID(), partitioningIdB, measureDefinitionIdB, OffsetDateTime.parse("2026-02-01T00:00:00Z"), "e1") + val mar = TestCheckpoint(UUID.randomUUID(), partitioningIdA, measureDefinitionIdA, OffsetDateTime.parse("2026-03-01T00:00:00Z"), "e2") + val apr = TestCheckpoint(UUID.randomUUID(), partitioningIdB, measureDefinitionIdB, OffsetDateTime.parse("2026-04-01T00:00:00Z"), "e3") + Seq(jan, feb, mar, apr).foreach { checkpoint => + table("runs.checkpoints").insert( + add("id_checkpoint", checkpoint.id) + .add("fk_partitioning", checkpoint.partitioningId) + .add("checkpoint_name", "CheckpointName") + .add("measured_by_atum_agent", true) + .add("process_start_time", checkpoint.start) + .add("created_by", "Joseph") + ) + table("runs.measurements").insert( + add("fk_measure_definition", checkpoint.measureDefinitionId) + .add("fk_checkpoint", checkpoint.id) + .add("measurement_value", measurementCnt) + ) + table("runs.checkpoint_properties").insert( + add("fk_checkpoint", checkpoint.id) + .add("property_name", "executionID") + .add("property_value", checkpoint.executionId) + ) + } + + def checkpointsOf(queryResult: QueryResult): Seq[(UUID, Long)] = + queryResult.map(row => (row.getUUID("id_checkpoint").get, row.getLong("id_partitioning").get)).toList + + // [feb, apr) spans both partitionings of the flow, latest first + function(fncGetFlowCheckpointsV2) + .setParam("i_flow_id", flowId) + .setParam("i_from_time", feb.start) + .setParam("i_to_time", apr.start) + .execute { queryResult => + assert(checkpointsOf(queryResult) == Seq((mar.id, partitioningIdA), (feb.id, partitioningIdB))) + } + + // the same window, earliest first + function(fncGetFlowCheckpointsV2) + .setParam("i_flow_id", flowId) + .setParam("i_from_time", feb.start) + .setParam("i_to_time", apr.start) + .setParam("i_latest_first", false) + .execute { queryResult => + assert(checkpointsOf(queryResult).map(_._1) == Seq(feb.id, mar.id)) + } + + // combined with a multi-value checkpoint properties filter: executionID IN (e1, e3) since February + function(fncGetFlowCheckpointsV2) + .setParam("i_flow_id", flowId) + .setParam("i_checkpoint_properties", JsonBString("""{"executionID": ["e1", "e3"]}""")) + .setParam("i_from_time", feb.start) + .execute { queryResult => + assert(checkpointsOf(queryResult).map(_._1) == Seq(apr.id, feb.id)) + } + + // paging within the window: has_more is computed over the window only + function(fncGetFlowCheckpointsV2) + .setParam("i_flow_id", flowId) + .setParam("i_checkpoints_limit", 1) + .setParam("i_from_time", feb.start) + .setParam("i_to_time", apr.start) + .execute { queryResult => + val row = queryResult.next() + assert(row.getUUID("id_checkpoint").contains(mar.id)) + assert(row.getBoolean("has_more").contains(true)) + assert(!queryResult.hasNext) + } + function(fncGetFlowCheckpointsV2) + .setParam("i_flow_id", flowId) + .setParam("i_checkpoints_limit", 1) + .setParam("i_offset", 1L) + .setParam("i_from_time", feb.start) + .setParam("i_to_time", apr.start) + .execute { queryResult => + val row = queryResult.next() + assert(row.getUUID("id_checkpoint").contains(feb.id)) + assert(row.getBoolean("has_more").contains(false)) + assert(!queryResult.hasNext) + } + } + private def parseJsonBStringOrThrow(jsonBString: JsonBString): Json = { parse(jsonBString.value).getOrElse(throw new Exception("Failed to parse JsonBString to Json")) } diff --git a/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala index 797b9d77b..110a961dd 100644 --- a/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.database.runs import za.co.absa.balta.DBTestSuite -import za.co.absa.balta.classes.JsonBString +import za.co.absa.balta.classes.{JsonBString, QueryResult} import za.co.absa.balta.classes.setter.CustomDBType import java.time.OffsetDateTime @@ -492,4 +492,118 @@ class GetPartitioningCheckpointsIntegrationTests extends DBTestSuite { } } + test("Returns only checkpoints within the process start time window") { + table("runs.partitionings").insert( + add("partitioning", partitioning1) + .add("created_by", "Daniel") + ) + val fkPartitioning: Long = table("runs.partitionings") + .fieldValue("partitioning", partitioning1, "id_partitioning").get.get + + table("runs.measure_definitions").insert( + add("id_measure_definition", id_measure_definition1) + .add("fk_partitioning", fkPartitioning) + .add("created_by", "Daniel") + .add("measure_name", "measure_1") + .add("measured_columns", measured_columns1) + ) + + // one checkpoint at the start of each month, January to April; named "odd" / "even" by month number + val times = Seq( + OffsetDateTime.parse("2026-01-01T00:00:00Z"), + OffsetDateTime.parse("2026-02-01T00:00:00Z"), + OffsetDateTime.parse("2026-03-01T00:00:00Z"), + OffsetDateTime.parse("2026-04-01T00:00:00Z") + ) + val checkpointIds = times.map(_ => UUID.randomUUID()) + times.zip(checkpointIds).zipWithIndex.foreach { case ((time, checkpointId), i) => + table("runs.checkpoints").insert( + add("id_checkpoint", checkpointId) + .add("fk_partitioning", fkPartitioning) + .add("checkpoint_name", if (i % 2 == 0) "odd" else "even") + .add("process_start_time", time) + .add("process_end_time", endTime) + .add("measured_by_atum_agent", true) + .add("created_by", "Daniel") + ) + table("runs.measurements").insert( + add("fk_checkpoint", checkpointId) + .add("fk_measure_definition", id_measure_definition1) + .add("measurement_value", measurement1) + ) + } + val Seq(jan, feb, mar, apr) = checkpointIds + + def checkpointsOf(queryResult: QueryResult): Seq[UUID] = + queryResult.map(_.getUUID("id_checkpoint").get).toList + + // i_from_time is inclusive + function(fncGetPartitioningCheckpoints) + .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_from_time", times(1)) + .execute { queryResult => + assert(checkpointsOf(queryResult) == Seq(apr, mar, feb)) + } + + // i_to_time is exclusive + function(fncGetPartitioningCheckpoints) + .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_to_time", times(2)) + .execute { queryResult => + assert(checkpointsOf(queryResult) == Seq(feb, jan)) + } + + // both bounds, earliest first + function(fncGetPartitioningCheckpoints) + .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_from_time", times(1)) + .setParam("i_to_time", times(3)) + .setParam("i_latest_first", false) + .execute { queryResult => + assert(checkpointsOf(queryResult) == Seq(feb, mar)) + } + + // paging within the window: has_more is computed over the window only + function(fncGetPartitioningCheckpoints) + .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_checkpoints_limit", 1) + .setParam("i_from_time", times(1)) + .setParam("i_to_time", times(3)) + .execute { queryResult => + val row = queryResult.next() + assert(row.getUUID("id_checkpoint").contains(mar)) + assert(row.getBoolean("has_more").contains(true)) + assert(!queryResult.hasNext) + } + function(fncGetPartitioningCheckpoints) + .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_checkpoints_limit", 1) + .setParam("i_offset", 1L) + .setParam("i_from_time", times(1)) + .setParam("i_to_time", times(3)) + .execute { queryResult => + val row = queryResult.next() + assert(row.getUUID("id_checkpoint").contains(feb)) + assert(row.getBoolean("has_more").contains(false)) + assert(!queryResult.hasNext) + } + + // combined with the checkpoint name filter + function(fncGetPartitioningCheckpoints) + .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_checkpoint_name", "odd") + .setParam("i_from_time", times(1)) + .execute { queryResult => + assert(checkpointsOf(queryResult) == Seq(mar)) + } + + // a window without checkpoints returns no rows + function(fncGetPartitioningCheckpoints) + .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_from_time", OffsetDateTime.parse("2027-01-01T00:00:00Z")) + .execute { queryResult => + assert(!queryResult.hasNext) + } + } + } From 29e28271ef84a415105efae5c0d9863dd025935c Mon Sep 17 00:00:00 2001 From: Ladislav Sulak Date: Fri, 9 Oct 2026 18:20:32 +0200 Subject: [PATCH 11/21] fix: reject checkpoint properties without any accepted value A property with an empty set of values (e.g. `{"executionID": []}`) silently matched no checkpoint. It is now rejected up front: CheckpointFilter throws an IllegalArgumentException on construction, and the server answers 400 Bad Request for such a `checkpoint-properties` query parameter. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../atum/reader/requests/CheckpointFilter.scala | 9 ++++++++- .../requests/CheckpointFilterUnitTests.scala | 8 ++++++++ .../absa/atum/server/api/v2/http/Endpoints.scala | 13 +++++++++++-- .../http/GetFlowCheckpointsEndpointUnitTests.scala | 14 ++++++++++++++ ...tPartitioningCheckpointsEndpointUnitTests.scala | 14 ++++++++++++++ 5 files changed, 55 insertions(+), 3 deletions(-) diff --git a/reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala b/reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala index 5ff07c8ff..f153a84e0 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala @@ -33,13 +33,20 @@ import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ * * @param name - only checkpoints of this name * @param properties - only checkpoints that, for every property name given, have that property with one of the given - * values, e.g. `Map("executionID" -> Set("a", "b"))` means `executionID IN (a, b)` + * values, e.g. `Map("executionID" -> Set("a", "b"))` means `executionID IN (a, b)`; every property + * needs at least one value + * @throws IllegalArgumentException if a property is given without any value */ case class CheckpointFilter( name: Option[String] = None, properties: Map[String, Set[String]] = Map.empty ) { + require( + properties.values.forall(_.nonEmpty), + s"Checkpoint properties without any accepted value: ${properties.collect { case (n, v) if v.isEmpty => n }.mkString(", ")}" + ) + private[reader] def toQueryParams: Map[String, String] = { name.map(QueryParamNames.CheckpointName -> _).toMap ++ propertiesParam } diff --git a/reader/src/test/scala/za/co/absa/atum/reader/requests/CheckpointFilterUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/requests/CheckpointFilterUnitTests.scala index 93b0cbad4..9c4ed8ee7 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/requests/CheckpointFilterUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/requests/CheckpointFilterUnitTests.scala @@ -54,4 +54,12 @@ class CheckpointFilterUnitTests extends AnyFunSuiteLike { ) } + test("A property without any value is rejected, as it would match no checkpoint") { + val exception = intercept[IllegalArgumentException] { + CheckpointFilter(properties = Map("executionID" -> Set.empty[String], "env" -> Set("prod"))) + } + assert(exception.getMessage.contains("executionID")) + assert(!exception.getMessage.contains("env")) + } + } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala index 906a22cc0..02df5a65d 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala @@ -20,7 +20,7 @@ import sttp.model.StatusCode import sttp.tapir.generic.auto.schemaForCaseClass import sttp.tapir.json.circe.jsonBody import sttp.tapir.ztapir._ -import sttp.tapir.{Codec, CodecFormat, DecodeResult, PublicEndpoint, Validator} +import sttp.tapir.{Codec, CodecFormat, DecodeResult, PublicEndpoint, ValidationResult, Validator} import za.co.absa.atum.model.ApiPaths._ import za.co.absa.atum.model.dto._ import za.co.absa.atum.model.envelopes.ErrorResponse @@ -35,7 +35,8 @@ import scala.util.{Failure, Success, Try} object Endpoints extends BaseEndpoints { // Checkpoint properties are supplied as a single query parameter carrying a base64url-encoded JSON object, - // e.g. `checkpoint-properties=eyJleGVjdXRpb25JRCI6Ii4uLiJ9`. Malformed base64 or JSON results in a 400 Bad Request. + // e.g. `checkpoint-properties=eyJleGVjdXRpb25JRCI6Ii4uLiJ9`. Malformed base64 or JSON, or a property without any + // accepted value (e.g. `{"executionID": []}`, which would match no checkpoint), results in a 400 Bad Request. private implicit val checkpointPropertiesQueryCodec: Codec[String, Map[String, Seq[String]], CodecFormat.TextPlain] = Codec.string.mapDecode { encoded => Try(encoded.fromBase64As[Map[String, Seq[String]]]) match { @@ -48,6 +49,14 @@ object Endpoints extends BaseEndpoints { } } }(_.asBase64EncodedJsonString) + .validate(Validator.custom { properties => + val propertiesWithoutValues = properties.collect { case (name, values) if values.isEmpty => name } + if (propertiesWithoutValues.isEmpty) ValidationResult.Valid + else + ValidationResult.Invalid( + s"Checkpoint properties without any accepted value: ${propertiesWithoutValues.mkString(", ")}" + ) + }) val postCheckpointEndpoint : PublicEndpoint[(Long, CheckpointV2DTO), ErrorResponse, (SingleSuccessResponse[CheckpointV2DTO], String), Any] = { diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala index be500cc6e..36297a654 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala @@ -186,6 +186,20 @@ object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) }, + test("Returns expected 400 when a checkpoint property has no accepted value") { + import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ + val encodedProps = Map("executionID" -> Seq.empty[String]).asBase64EncodedJsonString + val baseUri = + uri"https://test.com/api/v2/flows/1/checkpoints?limit=5&offset=0&checkpoint-properties=$encodedProps" + val response = basicRequest + .get(baseUri) + .response(asJson[PaginatedResponse[CheckpointWithPartitioningDTO]]) + .send(backendStub) + + val statusCode = response.map(_.code) + + assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) + }, test("Returns expected 400 when limit is out of range") { val baseUri = uri"https://test.com/api/v2/flows/1/checkpoints?limit=1005&offset=0" val response = basicRequest diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala index 9f5e0d4ad..1afa01ac5 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala @@ -211,6 +211,20 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) }, + test("Returns expected 400 when a checkpoint property has no accepted value") { + import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ + val encodedProps = Map("executionID" -> Seq.empty[String]).asBase64EncodedJsonString + val request = basicRequest + .get(uri"https://test.com/api/v2/partitionings/1/checkpoints?limit=10&offset=0&checkpoint-properties=$encodedProps") + .response(asJson[PaginatedResponse[CheckpointV2DTO]]) + + val response = request + .send(backendStub) + + val statusCode = response.map(_.code) + + assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) + }, test( "Returns an expected PaginatedResponse[CheckpointV2DTO] with more data available " + "(call without optional params)" From 36817e686f8a7300315628bfeda8dc93698cd889 Mon Sep 17 00:00:00 2001 From: Ladislav Sulak Date: Fri, 9 Oct 2026 18:41:09 +0200 Subject: [PATCH 12/21] fix: scaladoc link and coverage of empty checkpoint properties validation The `@throws IllegalArgumentException` tag could not be resolved by Scaladoc, failing `sbt doc` under -Xfatal-warnings; the exception is now described in the `properties` parameter doc. Replaced the partial-function `collect` with `filter` to avoid untestable generated bytecode lowering server coverage below threshold. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../za/co/absa/atum/reader/requests/CheckpointFilter.scala | 5 ++--- .../scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala b/reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala index f153a84e0..a5a497a2d 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala @@ -34,8 +34,7 @@ import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ * @param name - only checkpoints of this name * @param properties - only checkpoints that, for every property name given, have that property with one of the given * values, e.g. `Map("executionID" -> Set("a", "b"))` means `executionID IN (a, b)`; every property - * needs at least one value - * @throws IllegalArgumentException if a property is given without any value + * needs at least one value, otherwise an `IllegalArgumentException` is thrown */ case class CheckpointFilter( name: Option[String] = None, @@ -44,7 +43,7 @@ case class CheckpointFilter( require( properties.values.forall(_.nonEmpty), - s"Checkpoint properties without any accepted value: ${properties.collect { case (n, v) if v.isEmpty => n }.mkString(", ")}" + s"Checkpoint properties without any accepted value: ${properties.filter(_._2.isEmpty).keys.mkString(", ")}" ) private[reader] def toQueryParams: Map[String, String] = { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala index 02df5a65d..719cb657d 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala @@ -50,7 +50,7 @@ object Endpoints extends BaseEndpoints { } }(_.asBase64EncodedJsonString) .validate(Validator.custom { properties => - val propertiesWithoutValues = properties.collect { case (name, values) if values.isEmpty => name } + val propertiesWithoutValues = properties.filter(_._2.isEmpty).keys if (propertiesWithoutValues.isEmpty) ValidationResult.Valid else ValidationResult.Invalid( From ba5c8eecd2dc1cab00fc6e07689d41157bf77b85 Mon Sep 17 00:00:00 2001 From: Ladislav Sulak Date: Fri, 9 Oct 2026 19:10:19 +0200 Subject: [PATCH 13/21] test: exercise combined-filter pagination against seeded data in server integration tests The server integration tests for flow and partitioning checkpoints used non-existent IDs, so the database functions returned "not found" before evaluating the name and property filters or the pagination. They now seed a partitioning (and its flow) with matching and non-matching checkpoints and assert both pages and the has-more flag. A new single-connection, single-transaction test transactor rolls back once the test ends, so the seeded data is visible to all calls within a test but never persisted. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../server/api/TestTransactorProvider.scala | 21 ++++ .../GetFlowCheckpointsIntegrationTests.scala | 96 +++++++++++++++++-- ...titioningCheckpointsIntegrationTests.scala | 81 +++++++++++++++- 3 files changed, 184 insertions(+), 14 deletions(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala b/server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala index 08df37af7..e38d0db6f 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala @@ -22,6 +22,8 @@ import za.co.absa.atum.server.config.PostgresConfig import zio._ import zio.interop.catz._ +import java.sql.DriverManager + object TestTransactorProvider { val layerWithoutRollback: ZLayer[Any, Config.Error, Transactor[Task]] = ZLayer { @@ -53,4 +55,23 @@ object TestTransactorProvider { } yield transactorWithRollback } + // All database calls share a single connection and a single transaction, rolled back once the layer is released. + // This allows seeding data with some database functions and reading them back with others within a test. + val layerWithSingleTransactionRollback: ZLayer[Any, Throwable, Transactor[Task]] = ZLayer.scoped { + for { + postgresConfig <- ZIO.config[PostgresConfig](PostgresConfig.config) + connection <- ZIO.acquireRelease( + ZIO.attemptBlocking { + val connection = DriverManager.getConnection( + s"jdbc:postgresql://${postgresConfig.serverName}:${postgresConfig.portNumber}/${postgresConfig.databaseName}", + postgresConfig.user, + postgresConfig.password + ) + connection.setAutoCommit(false) + connection + } + )(connection => ZIO.attemptBlocking { connection.rollback(); connection.close() }.orDie) + } yield Transactor.strategy.set(Transactor.fromConnection[Task](connection), Strategy.void) + } + } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala index 31147df05..6ee769eaf 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala @@ -16,17 +16,77 @@ package za.co.absa.atum.server.api.database.flows.functions +import za.co.absa.atum.model.ResultValueType +import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue +import za.co.absa.atum.model.dto._ import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider +import za.co.absa.atum.server.api.database.flows.functions.GetFlowCheckpoints.GetFlowCheckpointsArgs +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointArgs +import za.co.absa.atum.server.api.database.runs.functions.{ + CreatePartitioning, + GetPartitioningMainFlow, + WriteCheckpointV2 +} import za.co.absa.db.fadb.exceptions.DataNotFoundException import za.co.absa.db.fadb.status.FunctionStatus import zio._ import zio.test._ import zio.interop.catz.asyncInstance +import java.time.ZonedDateTime +import java.util.UUID + object GetFlowCheckpointsIntegrationTests extends ConfigProviderTest { + private val checkpointName = "Data written" + private val firstStartTime = ZonedDateTime.parse("2026-01-01T00:00:00Z") + + private def createPartitioning: ZIO[CreatePartitioning, Throwable, Long] = + ZIO + .serviceWithZIO[CreatePartitioning]( + _(PartitioningSubmitV2DTO(Seq(PartitionDTO("key", UUID.randomUUID().toString)), None, "author")) + ) + .flatMap(ZIO.fromEither(_)) + .map(_.data) + + private def getMainFlowId(partitioningId: Long): ZIO[GetPartitioningMainFlow, Throwable, Long] = + ZIO + .serviceWithZIO[GetPartitioningMainFlow](_(partitioningId)) + .flatMap(ZIO.fromEither(_)) + .flatMap(row => ZIO.getOrFail(row.data.map(_.id))) + + private def writeCheckpoint( + partitioningId: Long, + name: String, + executionId: Option[String], + startMinute: Long + ): ZIO[WriteCheckpointV2, Throwable, UUID] = { + val checkpoint = CheckpointV2DTO( + id = UUID.randomUUID(), + name = name, + author = "author", + processStartTime = firstStartTime.plusMinutes(startMinute), + processEndTime = None, + measurements = Set( + MeasurementDTO(MeasureDTO("count", Seq("*")), MeasureResultDTO(TypedValue("1", ResultValueType.LongValue))) + ), + properties = executionId.map(id => Map("executionID" -> id)) + ) + ZIO + .serviceWithZIO[WriteCheckpointV2](_(WriteCheckpointArgs(partitioningId, checkpoint))) + .flatMap(ZIO.fromEither(_)) + .as(checkpoint.id) + } + + // returns the IDs of the checkpoints on the page, in the order received, and the has-more flag + private def getPage(args: GetFlowCheckpointsArgs): ZIO[GetFlowCheckpoints, Throwable, (Seq[UUID], Boolean)] = + ZIO.serviceWithZIO[GetFlowCheckpoints](_(args)).flatMap(ZIO.fromEither(_)).map { rows => + val items = rows.flatMap(_.data) + (items.map(_.idCheckpoint), items.exists(_.hasMore)) + } + override def spec: Spec[TestEnvironment with Scope, Any] = { suite("GetFlowCheckpointsIntegrationTests")( @@ -46,23 +106,39 @@ object GetFlowCheckpointsIntegrationTests extends ConfigProviderTest { } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(42, "Flow not found")))) }, test("Should apply pagination (limit and offset) accurately with combined filters") { - val args = GetFlowCheckpoints.GetFlowCheckpointsArgs( - flowId = 1L, - limit = 10, - offset = 0L, - checkpointName = Some("TestCheckpointName"), - checkpointProperties = Some(Map("key1" -> Seq("value1", "value2"))) + def getFilteredPage(flowId: Long, offset: Long) = getPage( + GetFlowCheckpointsArgs( + flowId = flowId, + limit = 2, + offset = offset, + checkpointName = Some(checkpointName), + checkpointProperties = Some(Map("executionID" -> Seq("a", "b"))) + ) ) for { - getFlowCheckpoints <- ZIO.service[GetFlowCheckpoints] - result <- getFlowCheckpoints(args) - } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(42, "Flow not found")))) + partitioningId <- createPartitioning + flowId <- getMainFlowId(partitioningId) + matching1 <- writeCheckpoint(partitioningId, checkpointName, Some("a"), startMinute = 1) + matching2 <- writeCheckpoint(partitioningId, checkpointName, Some("b"), startMinute = 2) + _ <- writeCheckpoint(partitioningId, checkpointName, Some("c"), startMinute = 3) + _ <- writeCheckpoint(partitioningId, "Other checkpoint", Some("a"), startMinute = 4) + _ <- writeCheckpoint(partitioningId, checkpointName, None, startMinute = 5) + matching3 <- writeCheckpoint(partitioningId, checkpointName, Some("a"), startMinute = 6) + firstPage <- getFilteredPage(flowId, offset = 0L) + secondPage <- getFilteredPage(flowId, offset = 2L) + } yield assertTrue( + firstPage == (Seq(matching3, matching2), true), + secondPage == (Seq(matching1), false) + ) } ).provide( GetFlowCheckpoints.layer, + CreatePartitioning.layer, + GetPartitioningMainFlow.layer, + WriteCheckpointV2.layer, PostgresDatabaseProvider.layer, - TestTransactorProvider.layerWithRollback + TestTransactorProvider.layerWithSingleTransactionRollback ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala index 3414b9cfb..efb2ebefc 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala @@ -16,18 +16,68 @@ package za.co.absa.atum.server.api.database.runs.functions +import za.co.absa.atum.model.ResultValueType +import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue +import za.co.absa.atum.model.dto._ import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.functions.GetPartitioningCheckpoints.GetPartitioningCheckpointsArgs +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointArgs import za.co.absa.db.fadb.exceptions.DataNotFoundException import za.co.absa.db.fadb.status.FunctionStatus import zio.interop.catz.asyncInstance import zio.{Scope, ZIO} import zio.test._ +import java.time.ZonedDateTime +import java.util.UUID + object GetPartitioningCheckpointsIntegrationTests extends ConfigProviderTest { + private val checkpointName = "Data written" + private val firstStartTime = ZonedDateTime.parse("2026-01-01T00:00:00Z") + + private def createPartitioning: ZIO[CreatePartitioning, Throwable, Long] = + ZIO + .serviceWithZIO[CreatePartitioning]( + _(PartitioningSubmitV2DTO(Seq(PartitionDTO("key", UUID.randomUUID().toString)), None, "author")) + ) + .flatMap(ZIO.fromEither(_)) + .map(_.data) + + private def writeCheckpoint( + partitioningId: Long, + name: String, + executionId: Option[String], + startMinute: Long + ): ZIO[WriteCheckpointV2, Throwable, UUID] = { + val checkpoint = CheckpointV2DTO( + id = UUID.randomUUID(), + name = name, + author = "author", + processStartTime = firstStartTime.plusMinutes(startMinute), + processEndTime = None, + measurements = Set( + MeasurementDTO(MeasureDTO("count", Seq("*")), MeasureResultDTO(TypedValue("1", ResultValueType.LongValue))) + ), + properties = executionId.map(id => Map("executionID" -> id)) + ) + ZIO + .serviceWithZIO[WriteCheckpointV2](_(WriteCheckpointArgs(partitioningId, checkpoint))) + .flatMap(ZIO.fromEither(_)) + .as(checkpoint.id) + } + + // returns the IDs of the checkpoints on the page, in the order received, and the has-more flag + private def getPage( + args: GetPartitioningCheckpointsArgs + ): ZIO[GetPartitioningCheckpoints, Throwable, (Seq[UUID], Boolean)] = + ZIO.serviceWithZIO[GetPartitioningCheckpoints](_(args)).flatMap(ZIO.fromEither(_)).map { rows => + val items = rows.flatMap(_.data) + (items.map(_.idCheckpoint), items.exists(_.hasMore)) + } + override def spec: Spec[TestEnvironment with Scope, Any] = { suite("GetPartitioningCheckpointsIntegrationTests")( test("Returns expected sequence of Checkpoints with non-existing partitioning id") { @@ -37,15 +87,38 @@ object GetPartitioningCheckpointsIntegrationTests extends ConfigProviderTest { } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) }, test("Should apply pagination (limit and offset) accurately with combined filters") { + def getFilteredPage(partitioningId: Long, offset: Long) = getPage( + GetPartitioningCheckpointsArgs( + partitioningId = partitioningId, + limit = 2, + offset = offset, + checkpointName = Some(checkpointName), + checkpointProperties = Some(Map("executionID" -> Seq("a", "b"))), + latestFirst = None + ) + ) + for { - getPartitioningCheckpoints <- ZIO.service[GetPartitioningCheckpoints] - result <- getPartitioningCheckpoints(GetPartitioningCheckpointsArgs(0L, 10, 0L, Some("TestCheckpointName"), Some(Map("key1" -> Seq("value1", "value2"))), None)) - } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) + partitioningId <- createPartitioning + matching1 <- writeCheckpoint(partitioningId, checkpointName, Some("a"), startMinute = 1) + matching2 <- writeCheckpoint(partitioningId, checkpointName, Some("b"), startMinute = 2) + _ <- writeCheckpoint(partitioningId, checkpointName, Some("c"), startMinute = 3) + _ <- writeCheckpoint(partitioningId, "Other checkpoint", Some("a"), startMinute = 4) + _ <- writeCheckpoint(partitioningId, checkpointName, None, startMinute = 5) + matching3 <- writeCheckpoint(partitioningId, checkpointName, Some("a"), startMinute = 6) + firstPage <- getFilteredPage(partitioningId, offset = 0L) + secondPage <- getFilteredPage(partitioningId, offset = 2L) + } yield assertTrue( + firstPage == (Seq(matching3, matching2), true), + secondPage == (Seq(matching1), false) + ) } ).provide( GetPartitioningCheckpoints.layer, + CreatePartitioning.layer, + WriteCheckpointV2.layer, PostgresDatabaseProvider.layer, - TestTransactorProvider.layerWithRollback + TestTransactorProvider.layerWithSingleTransactionRollback ) } From ae02cedd09cdd33f11c9e74b17880e9e742c7183 Mon Sep 17 00:00:00 2001 From: Ladislav Sulak Date: Fri, 9 Oct 2026 20:47:25 +0200 Subject: [PATCH 14/21] fix: keep the default checkpoints limit of 5 in the checkpoints DB functions The new overloads of flows.get_flow_checkpoints and runs.get_partitioning_checkpoints defaulted i_checkpoints_limit to NULL (all checkpoints) instead of the established 5, changing the behavior for direct callers omitting the argument. An explicit NULL still returns all. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/main/postgres/flows/V0.8.0.1__get_flow_checkpoints.sql | 2 +- .../postgres/runs/V0.8.0.2__get_partitioning_checkpoints.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/database/src/main/postgres/flows/V0.8.0.1__get_flow_checkpoints.sql b/database/src/main/postgres/flows/V0.8.0.1__get_flow_checkpoints.sql index 74c90e200..4ce2a4e65 100644 --- a/database/src/main/postgres/flows/V0.8.0.1__get_flow_checkpoints.sql +++ b/database/src/main/postgres/flows/V0.8.0.1__get_flow_checkpoints.sql @@ -20,7 +20,7 @@ DROP FUNCTION IF EXISTS flows.get_flow_checkpoints(BIGINT, INT, BIGINT, TEXT, HS CREATE OR REPLACE FUNCTION flows.get_flow_checkpoints( IN i_flow_id BIGINT, - IN i_checkpoints_limit INT DEFAULT NULL, + IN i_checkpoints_limit INT DEFAULT 5, IN i_offset BIGINT DEFAULT 0, IN i_checkpoint_name TEXT DEFAULT NULL, IN i_checkpoint_properties JSONB DEFAULT NULL, diff --git a/database/src/main/postgres/runs/V0.8.0.2__get_partitioning_checkpoints.sql b/database/src/main/postgres/runs/V0.8.0.2__get_partitioning_checkpoints.sql index 76ad85905..3ccc4103c 100644 --- a/database/src/main/postgres/runs/V0.8.0.2__get_partitioning_checkpoints.sql +++ b/database/src/main/postgres/runs/V0.8.0.2__get_partitioning_checkpoints.sql @@ -20,7 +20,7 @@ DROP FUNCTION IF EXISTS runs.get_partitioning_checkpoints(BIGINT, INT, BIGINT, T CREATE OR REPLACE FUNCTION runs.get_partitioning_checkpoints( IN i_partitioning_id BIGINT, - IN i_checkpoints_limit INT DEFAULT NULL, + IN i_checkpoints_limit INT DEFAULT 5, IN i_offset BIGINT DEFAULT 0, IN i_checkpoint_name TEXT DEFAULT NULL, IN i_checkpoint_properties JSONB DEFAULT NULL, From 28ef69aef949a484dac198f23db626156ba5310b Mon Sep 17 00:00:00 2001 From: Ladislav Sulak Date: Fri, 9 Oct 2026 20:49:39 +0200 Subject: [PATCH 15/21] test: close the single-transaction test connection even if rollback fails Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../za/co/absa/atum/server/api/TestTransactorProvider.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala b/server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala index e38d0db6f..98fa70435 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala @@ -70,7 +70,7 @@ object TestTransactorProvider { connection.setAutoCommit(false) connection } - )(connection => ZIO.attemptBlocking { connection.rollback(); connection.close() }.orDie) + )(connection => ZIO.attemptBlocking(try connection.rollback() finally connection.close()).orDie) } yield Transactor.strategy.set(Transactor.fromConnection[Task](connection), Strategy.void) } From bfd01f55c47ef7b7943f3555645f341918bf99cf Mon Sep 17 00:00:00 2001 From: Ladislav Sulak Date: Wed, 14 Oct 2026 14:46:03 +0200 Subject: [PATCH 16/21] this test is testing much less actually / there is quite an overlap, so removing it altogether --- .../GetFlowCheckpointsIntegrationTests.scala | 179 ------------------ 1 file changed, 179 deletions(-) diff --git a/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala index f3c71a747..0ff920d9c 100644 --- a/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala @@ -521,185 +521,6 @@ class GetFlowCheckpointsIntegrationTests extends DBTestSuite { } } - test("getFlowCheckpointsV2 should return limited with checkpoints for a given flow") { - - val partitioningId: Long = Random.nextLong() - table("runs.partitionings").insert( - add("id_partitioning", partitioningId) - .add("partitioning", partitioning) - .add("created_by", "Joseph") - ) - - val flowId: Long = Random.nextLong() - table("flows.flows").insert( - add("id_flow", flowId) - .add("flow_name", "flowName") - .add("from_pattern", false) - .add("created_by", "Joseph") - .add("fk_primary_partitioning", partitioningId) - ) - - table("flows.partitioning_to_flow").insert( - add("fk_flow", flowId) - .add("fk_partitioning", partitioningId) - .add("created_by", "ObviouslySomeTest") - ) - - // Insert checkpoints and measure definitions - // checkpoint1 has a later start time so it sorts first with latest_first=TRUE - val checkpointId1 = UUID.randomUUID() - val startTime = OffsetDateTime.parse("1993-02-14T10:00:00Z") - val endTime = OffsetDateTime.parse("2024-04-24T10:00:00Z") - table("runs.checkpoints").insert( - add("id_checkpoint", checkpointId1) - .add("fk_partitioning", partitioningId) - .add("checkpoint_name", "CheckpointNameCntAndAvg") - .add("measured_by_atum_agent", true) - .add("process_start_time", startTime) - .add("process_end_time", endTime) - .add("created_by", "Joseph") - ) - - val checkpointId2 = UUID.randomUUID() - val startTimeOther = OffsetDateTime.parse("1992-02-14T10:00:00Z") - val endTimeOther = OffsetDateTime.parse("2023-04-24T10:00:00Z") - table("runs.checkpoints").insert( - add("id_checkpoint", checkpointId2) - .add("fk_partitioning", partitioningId) - .add("checkpoint_name", "CheckpointNameOther") - .add("measured_by_atum_agent", true) - .add("process_start_time", startTimeOther) - .add("process_end_time", endTimeOther) - .add("created_by", "Joseph") - ) - - // Insert measure definitions and measurements - val measureDefinitionAvgId: Long = Random.nextLong() - table("runs.measure_definitions").insert( - add("id_measure_definition", measureDefinitionAvgId) - .add("fk_partitioning", partitioningId) - .add("measure_name", "avg") - .add("measured_columns", CustomDBType("""{"a","b"}""", "TEXT[]")) - .add("created_by", "Joseph") - ) - - val measureDefinitionCntId: Long = Random.nextLong() - table("runs.measure_definitions").insert( - add("id_measure_definition", measureDefinitionCntId) - .add("fk_partitioning", partitioningId) - .add("measure_name", "cnt") - .add("measured_columns", CustomDBType("""{"col1"}""", "TEXT[]")) - .add("created_by", "Joseph") - ) - - val measureDefinitionOtherId: Long = Random.nextLong() - table("runs.measure_definitions").insert( - add("id_measure_definition", measureDefinitionOtherId) - .add("fk_partitioning", partitioningId) - .add("measure_name", "sum") - .add("measured_columns", CustomDBType("""{"colOther"}""", "TEXT[]")) - .add("created_by", "Joseph") - ) - - table("runs.measurements").insert( - add("fk_measure_definition", measureDefinitionCntId) - .add("fk_checkpoint", checkpointId1) - .add("measurement_value", measurementCnt) - ) - - table("runs.measurements").insert( - add("fk_measure_definition", measureDefinitionAvgId) - .add("fk_checkpoint", checkpointId1) - .add("measurement_value", measurementAvg) - ) - - table("runs.measurements").insert( - add("fk_measure_definition", measureDefinitionOtherId) - .add("fk_checkpoint", checkpointId2) - .add("measurement_value", measurementSum) - ) - - // Actual test execution and assertions - val actualMeasures: Seq[MeasuredDetails] = function(fncGetFlowCheckpointsV2) - .setParam("i_flow_id", flowId) - .setParam("i_checkpoints_limit", 2) - .setParam("i_offset", 0L) - .execute("checkpoint_name") { queryResult => - assert(queryResult.hasNext) - val row1 = queryResult.next() - assert(row1.getInt("status").contains(11)) - assert(row1.getString("status_text").contains("OK")) - assert(row1.getUUID("id_checkpoint").contains(checkpointId1)) - assert(row1.getString("checkpoint_name").contains("CheckpointNameCntAndAvg")) - assert(row1.getString("checkpoint_author").contains("Joseph")) - assert(row1.getBoolean("measured_by_atum_agent").contains(true)) - assert(row1.getOffsetDateTime("checkpoint_start_time").contains(startTime)) - assert(row1.getOffsetDateTime("checkpoint_end_time").contains(endTime)) - assert(queryResult.hasNext) - - val measure1 = MeasuredDetails( - row1.getString("measure_name").get, - row1.getArray[String]("measured_columns").map(_.toList).get, - row1.getJsonB("measurement_value").get - ) - - val row2 = queryResult.next() - assert(row2.getInt("status").contains(11)) - assert(row2.getString("status_text").contains("OK")) - assert(row2.getUUID("id_checkpoint").contains(checkpointId1)) - assert(row2.getString("checkpoint_name").contains("CheckpointNameCntAndAvg")) - assert(row2.getString("checkpoint_author").contains("Joseph")) - assert(row2.getBoolean("measured_by_atum_agent").contains(true)) - assert(row2.getOffsetDateTime("checkpoint_start_time").contains(startTime)) - assert(row2.getOffsetDateTime("checkpoint_end_time").contains(endTime)) - assert(queryResult.hasNext) - - val measure2 = MeasuredDetails( - row2.getString("measure_name").get, - row2.getArray[String]("measured_columns").map(_.toList).get, - row2.getJsonB("measurement_value").get - ) - - val row3 = queryResult.next() - assert(row3.getInt("status").contains(11)) - assert(row3.getString("status_text").contains("OK")) - assert(row3.getUUID("id_checkpoint").contains(checkpointId2)) - assert(row3.getString("checkpoint_name").contains("CheckpointNameOther")) - assert(row3.getString("checkpoint_author").contains("Joseph")) - assert(row3.getBoolean("measured_by_atum_agent").contains(true)) - assert(row3.getOffsetDateTime("checkpoint_start_time").contains(startTimeOther)) - assert(row3.getOffsetDateTime("checkpoint_end_time").contains(endTimeOther)) - - val measure3 = MeasuredDetails( - row3.getString("measure_name").get, - row3.getArray[String]("measured_columns").map(_.toList).get, - row3.getJsonB("measurement_value").get - ) - - assert(!queryResult.hasNext) - Seq(measure1, measure2, measure3) - } - - // Assertions for measures - assert(actualMeasures.map(_.measureName).toSet == Set("avg", "sum")) - assert(actualMeasures.map(_.measureColumns).toSet == Set(List("a", "b"), List("colOther"))) - - actualMeasures.foreach { currVal => - val currValStr = currVal.measurementValue.value - - currVal.measureName match { - case "avg" => - assert(currValStr.contains(""""value": "2.71"""")) - case "cnt" => - assert(currValStr.contains(""""value": "3"""")) - case "sum" => - assert(currValStr.contains(""""value": "3000"""")) - case other => - fail(s"Unexpected measure name: $other") - } - } - } - test("getFlowCheckpointsV2 should return no flows when flow_id is not found") { // Create a non-existent flowId that doesn't exist in the database From e1d4a988a5f0199844d56bf9185a8b04002de5b8 Mon Sep 17 00:00:00 2001 From: Ladislav Sulak Date: Wed, 14 Oct 2026 14:50:20 +0200 Subject: [PATCH 17/21] redundant --- ...GetPartitioningCheckpointsIntegrationTests.scala | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala index 110a961dd..aefd225e5 100644 --- a/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala @@ -280,19 +280,6 @@ class GetPartitioningCheckpointsIntegrationTests extends DBTestSuite { assert(!queryResult.hasNext) } - function(fncGetPartitioningCheckpoints) - .setParam("i_partitioning_id", 0L) - .setParam("i_checkpoints_limit", 2) - .setParam("i_offset", 1) - .setParam("i_checkpoint_name", i_checkpoint_name) - .execute { queryResult => - assert(queryResult.hasNext) - val result = queryResult.next() - assert(result.getInt("status").contains(41)) - assert(result.getString("status_text").contains("Partitioning not found")) - assert(!queryResult.hasNext) - } - function(fncGetPartitioningCheckpoints) .setParam("i_partitioning_id", fkPartitioning1) .setParam("i_checkpoints_limit", 1) From aaecb4a00c378e2a6385f1032ca68e764dfa2827 Mon Sep 17 00:00:00 2001 From: Ladislav Sulak Date: Wed, 14 Oct 2026 15:24:36 +0200 Subject: [PATCH 18/21] refactoring mostly --- .../GetFlowCheckpointsIntegrationTests.scala | 4 ++-- ...titioningCheckpointsIntegrationTests.scala | 21 ++++++++++++++----- .../GetFlowCheckpointsEndpointUnitTests.scala | 2 +- ...itioningCheckpointsEndpointUnitTests.scala | 3 +-- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala index 6ee769eaf..d21a68ece 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala @@ -90,9 +90,9 @@ object GetFlowCheckpointsIntegrationTests extends ConfigProviderTest { override def spec: Spec[TestEnvironment with Scope, Any] = { suite("GetFlowCheckpointsIntegrationTests")( - test("Should return checkpoints with the correct flowId, limit, and offset") { + test("Should return DataNotFoundException when flow does not exist") { - val args = GetFlowCheckpoints.GetFlowCheckpointsArgs( + val args = GetFlowCheckpointsArgs( flowId = 1L, limit = 10, offset = 0L, diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala index efb2ebefc..93fb53dbe 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala @@ -80,21 +80,30 @@ object GetPartitioningCheckpointsIntegrationTests extends ConfigProviderTest { override def spec: Spec[TestEnvironment with Scope, Any] = { suite("GetPartitioningCheckpointsIntegrationTests")( - test("Returns expected sequence of Checkpoints with non-existing partitioning id") { + test("Should return DataNotFoundException when partitioning does not exist") { + val args = GetPartitioningCheckpointsArgs( + partitioningId = 0L, + limit = 10, + offset = 0L, + checkpointName = None, + checkpointProperties = None, + latestFirst = None + ) + for { getPartitioningCheckpoints <- ZIO.service[GetPartitioningCheckpoints] - result <- getPartitioningCheckpoints(GetPartitioningCheckpointsArgs(0L, 10, 0L, None, None, None)) + result <- getPartitioningCheckpoints(args) } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) }, test("Should apply pagination (limit and offset) accurately with combined filters") { - def getFilteredPage(partitioningId: Long, offset: Long) = getPage( + def getFilteredPage(partitioningId: Long, offset: Long, latestFirst: Option[Boolean] = None) = getPage( GetPartitioningCheckpointsArgs( partitioningId = partitioningId, limit = 2, offset = offset, checkpointName = Some(checkpointName), checkpointProperties = Some(Map("executionID" -> Seq("a", "b"))), - latestFirst = None + latestFirst = latestFirst ) ) @@ -108,9 +117,11 @@ object GetPartitioningCheckpointsIntegrationTests extends ConfigProviderTest { matching3 <- writeCheckpoint(partitioningId, checkpointName, Some("a"), startMinute = 6) firstPage <- getFilteredPage(partitioningId, offset = 0L) secondPage <- getFilteredPage(partitioningId, offset = 2L) + earliestFirstPage <- getFilteredPage(partitioningId, offset = 0L, latestFirst = Some(false)) } yield assertTrue( firstPage == (Seq(matching3, matching2), true), - secondPage == (Seq(matching1), false) + secondPage == (Seq(matching1), false), + earliestFirstPage == (Seq(matching1, matching2), true) ) } ).provide( diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala index 36297a654..ef56c4d42 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala @@ -26,6 +26,7 @@ import sttp.tapir.ztapir.{RIOMonadError, RichZEndpoint} import za.co.absa.atum.model.dto.CheckpointWithPartitioningDTO import za.co.absa.atum.model.envelopes.SuccessResponse.PaginatedResponse import za.co.absa.atum.model.envelopes.{NotFoundErrorResponse, Pagination} +import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.v2.controller.FlowController import zio.test.Assertion.equalTo @@ -187,7 +188,6 @@ object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) }, test("Returns expected 400 when a checkpoint property has no accepted value") { - import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ val encodedProps = Map("executionID" -> Seq.empty[String]).asBase64EncodedJsonString val baseUri = uri"https://test.com/api/v2/flows/1/checkpoints?limit=5&offset=0&checkpoint-properties=$encodedProps" diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala index 1afa01ac5..41e320fb9 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala @@ -26,6 +26,7 @@ import sttp.tapir.ztapir.{RIOMonadError, RichZEndpoint} import za.co.absa.atum.model.dto.CheckpointV2DTO import za.co.absa.atum.model.envelopes.SuccessResponse.PaginatedResponse import za.co.absa.atum.model.envelopes.{NotFoundErrorResponse, Pagination} +import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.v2.controller.CheckpointController import zio.test.Assertion.equalTo @@ -95,7 +96,6 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T suite("GetPartitioningCheckpointsEndpointSuite")( test("Returns an expected PaginatedResponse[CheckpointV2DTO] with checkpoint-properties having multiple values") { - import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ val properties = Map("executionID" -> Seq("id1", "id2")) val encodedProps = properties.asBase64EncodedJsonString val request = basicRequest @@ -212,7 +212,6 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) }, test("Returns expected 400 when a checkpoint property has no accepted value") { - import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ val encodedProps = Map("executionID" -> Seq.empty[String]).asBase64EncodedJsonString val request = basicRequest .get(uri"https://test.com/api/v2/partitionings/1/checkpoints?limit=10&offset=0&checkpoint-properties=$encodedProps") From 683e108b4e4c2f6dfe1461ffa5eb62d7c830a23d Mon Sep 17 00:00:00 2001 From: Ladislav Sulak Date: Wed, 14 Oct 2026 15:30:05 +0200 Subject: [PATCH 19/21] improving code doc --- .../co/absa/atum/server/api/TestTransactorProvider.scala | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala b/server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala index 98fa70435..fb75f23ac 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala @@ -56,7 +56,13 @@ object TestTransactorProvider { } // All database calls share a single connection and a single transaction, rolled back once the layer is released. - // This allows seeding data with some database functions and reading them back with others within a test. + // + // Why: some tests first insert their own test data ("seed" it) by calling database functions, e.g. CreatePartitioning + // to create a partitioning and WriteCheckpointV2 to write checkpoints into it, and then call the function under test, + // e.g. GetPartitioningCheckpoints, to read that data back. For this to work, the inserted rows must still exist when + // the function under test runs: + // - layerWithRollback rolls back after every single call, so the inserted rows are gone before they can be read; + // - layerWithoutRollback would keep the rows, but they would also stay in the test database permanently. val layerWithSingleTransactionRollback: ZLayer[Any, Throwable, Transactor[Task]] = ZLayer.scoped { for { postgresConfig <- ZIO.config[PostgresConfig](PostgresConfig.config) From f74ac8838d0d6623ece612a9cf1b69377ad9052b Mon Sep 17 00:00:00 2001 From: Ladislav Sulak Date: Wed, 14 Oct 2026 15:31:13 +0200 Subject: [PATCH 20/21] improving code doc --- .../absa/atum/server/api/TestTransactorProvider.scala | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala b/server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala index fb75f23ac..39ade40d8 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala @@ -57,12 +57,10 @@ object TestTransactorProvider { // All database calls share a single connection and a single transaction, rolled back once the layer is released. // - // Why: some tests first insert their own test data ("seed" it) by calling database functions, e.g. CreatePartitioning - // to create a partitioning and WriteCheckpointV2 to write checkpoints into it, and then call the function under test, - // e.g. GetPartitioningCheckpoints, to read that data back. For this to work, the inserted rows must still exist when - // the function under test runs: - // - layerWithRollback rolls back after every single call, so the inserted rows are gone before they can be read; - // - layerWithoutRollback would keep the rows, but they would also stay in the test database permanently. + // Why: some tests first insert test data (e.g. via CreatePartitioning, WriteCheckpointV2) and then read it back + // with the function under test (e.g. GetPartitioningCheckpoints), so the inserted rows must survive between calls: + // - layerWithRollback rolls back after each call, so the rows are gone before they can be read; + // - layerWithoutRollback keeps the rows, but permanently. val layerWithSingleTransactionRollback: ZLayer[Any, Throwable, Transactor[Task]] = ZLayer.scoped { for { postgresConfig <- ZIO.config[PostgresConfig](PostgresConfig.config) From ab4383be7f656d2f97655c615684ab080e922979 Mon Sep 17 00:00:00 2001 From: Ladislav Sulak Date: Fri, 16 Oct 2026 15:44:37 +0200 Subject: [PATCH 21/21] #478 (2): checkpoint time window and latest-first on flows (audit consumption pattern) (#489) * feat: audit consumption pattern (ADR 02) - process start time window and latest-first on flows Builds on the #478 reader capability work and exposes what ADR 02 section 4 needs for audits: - server: optional `from` / `to` (half-open window on process_start_time, from < to validated) on GET /partitionings/{id}/checkpoints and GET /flows/{id}/checkpoints, and `latest-first` on the flows endpoint; wired through controllers, services, repositories and DB function wrappers - server: flow checkpoints keep the DB order instead of re-sorting by start time, so latest-first=false works - reader: CheckpointFilter.from / to / latestFirst, getAllCheckpoints on FlowReader and PartitioningReader (Reader.queryAllPages), new query parameter names - db: V0.8.0.4 composite index runs.checkpoints (fk_partitioning, process_start_time) for the time window - ADR 02 - consumption pattern for audits The DB function signatures (i_from_time / i_to_time) already landed with #478, so no function is re-created here. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix: stack-safe paging of all checkpoints and ADR 02 response shape queryAllPages recursed through flatMap, which for strict effects (e.g. Identity) runs the continuation right away, adding stack frames per page. The continuation now hands the collected records over to a tail-recursive loop when it runs synchronously, and recurses within the effect otherwise (asynchronous or lazy effects), the hand-over being atomic. ADR 02: a flow checkpoints page item is one checkpoint with its measurements nested (processStartTime, measurements, partitioning), and parent -> child edges cannot be reconstructed today (Bet 3). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fixing unit tests * reader will not need this method most likely, removing * finalization * Delete adrs/02-consumption-pattern-for-audits.md will be in another PR * fix(db): prevent null flow pagination metadata Move LIMIT/OFFSET inside the counted subquery so nonzero offsets cannot discard the aggregate row and return NULL for has_more. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Update README with Reader feature requirements Clarify requirements for Reader features and server compatibility. Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- README.md | 11 ++- agent/README.md | 15 ++- .../V0.8.0.5__get_flow_partitionings.sql | 98 +++++++++++++++++++ ...ckpoints_partitioning_start_time_index.ddl | 23 +++++ ...GetFlowPartitioningsIntegrationTests.scala | 66 +++++++++++++ .../za/co/absa/atum/reader/FlowReader.scala | 4 +- .../absa/atum/reader/PartitioningReader.scala | 4 +- .../reader/requests/CheckpointFilter.scala | 34 +++++-- .../reader/requests/QueryParamNames.scala | 3 + .../atum/reader/FlowReaderUnitTests.scala | 10 +- .../reader/PartitioningReaderUnitTests.scala | 8 +- .../requests/CheckpointFilterUnitTests.scala | 16 ++- .../flows/functions/GetFlowCheckpoints.scala | 12 ++- .../GetPartitioningCheckpoints.scala | 10 +- .../v2/controller/CheckpointController.scala | 3 + .../controller/CheckpointControllerImpl.scala | 5 + .../api/v2/controller/FlowController.scala | 5 + .../v2/controller/FlowControllerImpl.scala | 17 +++- .../atum/server/api/v2/http/Endpoints.scala | 37 +++++-- .../v2/repository/CheckpointRepository.scala | 3 + .../repository/CheckpointRepositoryImpl.scala | 14 ++- .../api/v2/repository/FlowRepository.scala | 5 + .../v2/repository/FlowRepositoryImpl.scala | 9 +- .../api/v2/service/CheckpointService.scala | 3 + .../v2/service/CheckpointServiceImpl.scala | 5 + .../server/api/v2/service/FlowService.scala | 5 + .../api/v2/service/FlowServiceImpl.scala | 17 +++- ...CheckpointItemWithPartitioningFromDB.scala | 6 +- .../GetFlowCheckpointsIntegrationTests.scala | 34 ++++++- ...GetFlowPartitioningsIntegrationTests.scala | 43 +++++++- ...titioningCheckpointsIntegrationTests.scala | 32 +++++- .../CheckpointControllerUnitTests.scala | 6 +- .../controller/FlowControllerUnitTests.scala | 12 +-- .../GetFlowCheckpointsEndpointUnitTests.scala | 88 +++++++++++++++-- ...itioningCheckpointsEndpointUnitTests.scala | 72 ++++++++++++-- .../CheckpointRepositoryUnitTests.scala | 14 +-- .../repository/FlowRepositoryUnitTests.scala | 14 +-- .../service/CheckpointServiceUnitTests.scala | 8 +- .../api/v2/service/FlowServiceUnitTests.scala | 8 +- ...tItemWithPartitioningFromDBUnitTests.scala | 22 +++-- 40 files changed, 692 insertions(+), 109 deletions(-) create mode 100644 database/src/main/postgres/flows/V0.8.0.5__get_flow_partitionings.sql create mode 100644 database/src/main/postgres/runs/V0.8.0.4__checkpoints_partitioning_start_time_index.ddl diff --git a/README.md b/README.md index 8e9880fb2..228a57284 100644 --- a/README.md +++ b/README.md @@ -108,12 +108,13 @@ drop support for Spark 2.4. If you need the agent to work with Spark 2.4 follow * Build these two modules and use them in your project ### Reader `reader/` -**NB!** -_This module is not yet implemented to an operational abilities and therefore not yet released._ +This module is a Scala client for applications reading metrics stored by the _Atum Service_. `FlowReader` and +`PartitioningReader` resolve partitionings and retrieve checkpoint pages through the server's REST API. +Applications control pagination; the Reader does not discover datasets from partial business keys or reconstruct +the original parent-child lineage graph. -This module is intended to be used whenever an application wants to read the metrics stored by the _Atum Service_. It -offers classes and methods to read the metrics from the database shielding away the complexity of accessing the _Atum Server_ -directly. +Reader features require a compatible server version. In particular, older servers can ignore checkpoint time-window +parameters; using a newer Reader alone does not enable server-side filtering. ### Server `server/` An API under construction that communicates with the Agent and with the persistent storage. It also provides measure diff --git a/agent/README.md b/agent/README.md index 70445690c..a85705f06 100644 --- a/agent/README.md +++ b/agent/README.md @@ -25,7 +25,7 @@ Use `AtumPartitions` to get an `AtumContext` from the service using the `AtumAge import za.co.absa.atum.agent.AtumAgent import za.co.absa.atum.model.types.basic.AtumPartitions -val atumPartitions = AtumPartitions("country" -> "za", "dataset" -> "customers") +val atumPartitions = AtumPartitions(List("country" -> "za", "dataset" -> "customers")) val atumContext1 = AtumAgent.getOrCreateAtumContext(atumPartitions) ``` @@ -45,7 +45,7 @@ val config = ConfigFactory.parseString( ) val customAgent = AtumAgent.fromConfig(config) -val atumPartitions = AtumPartitions("country" -> "za", "dataset" -> "customers") +val atumPartitions = AtumPartitions(List("country" -> "za", "dataset" -> "customers")) val atumContext = customAgent.getOrCreateAtumContext(atumPartitions) ``` @@ -53,11 +53,18 @@ val atumContext = customAgent.getOrCreateAtumContext(atumPartitions) A list of key values that maintains the order of arrival of the items, the `AtumService` is able to deliver the correct `AtumContext` according to the `AtumPartitions` we give it. ```scala - val atumPartitions = AtumPartitions().withPartitions(ListMap("name" -> "partition-name", "country" -> "SA", "gender" -> "female" )) +val atumPartitions = AtumPartitions(List( + "name" -> "partition-name", + "country" -> "SA", + "gender" -> "female" +)) - val subPartition = atumPartitions.addPartition("otherKey", "otherValue") +val withOtherKey = atumPartitions ++ AtumPartitions("otherKey" -> "otherValue") ``` +Constructing these keys does not register a parent-child relationship. Use +`AtumContext.subPartitionContext` to create a linked child context. + Control measures can also be overwritten, added or removed. ```scala diff --git a/database/src/main/postgres/flows/V0.8.0.5__get_flow_partitionings.sql b/database/src/main/postgres/flows/V0.8.0.5__get_flow_partitionings.sql new file mode 100644 index 000000000..fd9e5dd15 --- /dev/null +++ b/database/src/main/postgres/flows/V0.8.0.5__get_flow_partitionings.sql @@ -0,0 +1,98 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE OR REPLACE FUNCTION flows.get_flow_partitionings( + IN i_flow_id BIGINT, + IN i_limit INT DEFAULT 5, + IN i_offset BIGINT DEFAULT 0, + OUT status INTEGER, + OUT status_text TEXT, + OUT id BIGINT, + OUT partitioning JSONB, + OUT author TEXT, + OUT has_more BOOLEAN +) RETURNS SETOF record AS +------------------------------------------------------------------------------- +-- +-- Function: flows.get_flow_partitionings(3) +-- Retrieves all partitionings associated with the input flow. +-- +-- Note: partitionings are ordered by id_partitioning, then created_at descending. +-- +-- Parameters: +-- i_flow_id - flow id to use for identifying the partitionings that will be retrieved +-- i_limit - (optional) maximum number of partitionings to return, default is 5 +-- i_offset - (optional) offset to use for pagination, default is 0 +-- +-- Returns: +-- status - Status code +-- status_text - Status text +-- id - ID of retrieved partitioning +-- partitioning - Partitioning value +-- author - Author of the partitioning +-- has_more - Flag indicating if there are more partitionings available +-- +-- Status codes: +-- 11 - OK +-- 41 - Flow not found +-- +------------------------------------------------------------------------------- +$$ +DECLARE + _has_more BOOLEAN; +BEGIN + PERFORM 1 FROM flows.flows WHERE id_flow = i_flow_id; + IF NOT FOUND THEN + status := 41; + status_text := 'Flow not found'; + RETURN NEXT; + RETURN; + END IF; + + IF i_limit IS NOT NULL THEN + SELECT count(*) > i_limit + FROM (SELECT 1 + FROM flows.partitioning_to_flow PTF + WHERE PTF.fk_flow = i_flow_id + LIMIT i_limit + 1 OFFSET i_offset) s + INTO _has_more; + ELSE + _has_more := false; + END IF; + + + RETURN QUERY + SELECT + 11 AS status, + 'OK' AS status_text, + P.id_partitioning, + P.partitioning, + P.created_by, + _has_more + FROM + runs.partitionings P INNER JOIN + flows.partitioning_to_flow PF ON PF.fk_partitioning = P.id_partitioning + WHERE + PF.fk_flow = i_flow_id + ORDER BY + P.id_partitioning, + P.created_at DESC + LIMIT i_limit OFFSET i_offset; +END; +$$ +LANGUAGE plpgsql VOLATILE SECURITY DEFINER; + +GRANT EXECUTE ON FUNCTION flows.get_flow_partitionings(BIGINT, INT, BIGINT) TO atum_owner; diff --git a/database/src/main/postgres/runs/V0.8.0.4__checkpoints_partitioning_start_time_index.ddl b/database/src/main/postgres/runs/V0.8.0.4__checkpoints_partitioning_start_time_index.ddl new file mode 100644 index 000000000..9e7d84004 --- /dev/null +++ b/database/src/main/postgres/runs/V0.8.0.4__checkpoints_partitioning_start_time_index.ddl @@ -0,0 +1,23 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Supports the process start time window of runs.get_partitioning_checkpoints and flows.get_flow_checkpoints: +-- checkpoints of each partitioning are range-scanned by process_start_time. +-- +-- Built CONCURRENTLY so writes to runs.checkpoints are not blocked. This file must contain only this statement: +-- Flyway then runs it outside of a transaction, which CONCURRENTLY requires. +-- If the build fails, it leaves an INVALID index behind; drop it (DROP INDEX CONCURRENTLY runs.checkpoints_idx2) +-- and re-run the migration. +CREATE INDEX CONCURRENTLY checkpoints_idx2 ON runs.checkpoints (fk_partitioning, process_start_time); diff --git a/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowPartitioningsIntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowPartitioningsIntegrationTests.scala index 0a9745ce8..c8053fc6f 100644 --- a/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowPartitioningsIntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowPartitioningsIntegrationTests.scala @@ -174,6 +174,72 @@ class GetFlowPartitioningsIntegrationTests extends DBTestSuite { } } + Seq( + ("first page", Some(2), 0L, Seq(0, 1), true), + ("middle page", Some(2), 2L, Seq(2, 3), true), + ("last partial page", Some(2), 4L, Seq(4), false), + ("exact final full page", Some(2), 3L, Seq(3, 4), false), + ("offset at the end", Some(2), 5L, Seq.empty[Int], false), + ("offset beyond the end", Some(2), 6L, Seq.empty[Int], false), + ("unlimited first page", None, 0L, Seq(0, 1, 2, 3, 4), false), + ("unlimited page with an offset", None, 2L, Seq(2, 3, 4), false) + ).foreach { case (description, limit, offset, expectedIndices, hasMore) => + test(s"Returns ordered partitionings and has_more for $description") { + val partitionings = (0 until 5).map { index => + val partitioning = JsonBString( + s"""{"version":1,"keys":["page"],"keysToValuesMap":{"page":"$index"}}""" + ) + table(partitioningsTable).insert(add("partitioning", partitioning).add("created_by", "Pagination author")) + val id: Long = table(partitioningsTable) + .fieldValue("partitioning", partitioning, "id_partitioning").get.get + (id, partitioning) + }.sortBy(_._1) + + val flowId = function(createFlowFn) + .setParam("i_fk_partitioning", partitionings.last._1) + .setParam("i_by_user", "Pagination author") + .execute { queryResult => + val row = queryResult.next() + assert(row.getInt("status").contains(11)) + row.getLong("id_flow").get + } + + partitionings.init.reverse.foreach { case (id, _) => + function(addToParentFlowsFn) + .setParam("i_fk_parent_partitioning", partitionings.last._1) + .setParam("i_fk_partitioning", id) + .setParam("i_by_user", "Pagination author") + .execute { queryResult => + assert(queryResult.next().getInt("status").contains(11)) + } + } + + val pageQuery = function(getFlowPartitioningsFn) + .setParam("i_flow_id", flowId) + .setParam("i_offset", offset) + val limitedQuery = limit match { + case Some(value) => pageQuery.setParam("i_limit", value) + case None => pageQuery.setParamNull("i_limit") + } + + limitedQuery.execute { queryResult => + expectedIndices.foreach { index => + assert(queryResult.hasNext) + val row = queryResult.next() + val (expectedId, expectedPartitioning) = partitionings(index) + assert(row.getInt("status").contains(11)) + assert(row.getString("status_text").contains("OK")) + assert(row.getLong("id").contains(expectedId)) + assert(parseJsonBStringOrThrow(row.getJsonB("partitioning").get) == + parseJsonBStringOrThrow(expectedPartitioning)) + assert(row.getString("author").contains("Pagination author")) + assert(row.getBoolean("has_more").contains(hasMore)) + } + assert(!queryResult.hasNext) + } + } + } + test("Fails for non-existent flow"){ function(getFlowPartitioningsFn) .setParam("i_flow_id", 999999) diff --git a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala index 4e41985ec..57f104d95 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala @@ -45,12 +45,12 @@ case class FlowReader[F[_]](mainFlowPartitioning: AtumPartitions)(implicit /** * Function to retrieve a page of checkpoints belonging to the flow, optionally filtered. - * The checkpoints are ordered by their process start time, latest first. + * The checkpoints are ordered by their process start time, latest first unless the filter says otherwise. * * @param pageSize - the size of the page (record count) to be returned * @param offset - offset of the page (starting position) * @param includeProperties - whether to include checkpoint properties in the response - * @param filter - the checkpoints to return (by name and/or properties) + * @param filter - the checkpoints to return (name, properties, process start time window) and their order * @return - a page of checkpoints */ def getCheckpointsPage( diff --git a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala index 0346788cc..5a38732f1 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala @@ -43,12 +43,12 @@ case class PartitioningReader[F[_]](partitioning: AtumPartitions)(implicit /** * Function to retrieve a page of checkpoints belonging to the partitioning, optionally filtered. - * The checkpoints are ordered by their process start time, latest first. + * The checkpoints are ordered by their process start time, latest first unless the filter says otherwise. * * @param pageSize - the size of the page (record count) to be returned * @param offset - offset of the page (starting position) * @param includeProperties - whether to include checkpoint properties in the response - * @param filter - the checkpoints to return (by name and/or properties) + * @param filter - the checkpoints to return (name, properties, process start time window) and their order * @return - a page of checkpoints */ def getCheckpointsPage( diff --git a/reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala b/reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala index a5a497a2d..4f2c13e23 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala @@ -18,27 +18,37 @@ package za.co.absa.atum.reader.requests import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ +import java.time.ZonedDateTime + /** * Filter of a checkpoints query. All its parts are optional and a checkpoint has to satisfy all the given ones. * - * Example - checkpoints of the given name, of any of the two executions: + * Example - checkpoints of two executions processed in the last two months, latest first: * {{{ * CheckpointFilter( - * name = Some("Data written"), - * properties = Map("executionID" -> Set("a", "b")) + * properties = Map("executionID" -> Set("a", "b")), + * from = Some(ZonedDateTime.now().minusMonths(2)) * ) * }}} * - * Note: properties with more than one value require an Atum server supporting them; older servers reject them. + * Note: the time window and properties with more than one value require an Atum server supporting them; older servers + * ignore the time window and reject multi-value properties. * - * @param name - only checkpoints of this name - * @param properties - only checkpoints that, for every property name given, have that property with one of the given - * values, e.g. `Map("executionID" -> Set("a", "b"))` means `executionID IN (a, b)`; every property - * needs at least one value, otherwise an `IllegalArgumentException` is thrown + * @param name - only checkpoints of this name + * @param properties - only checkpoints that, for every property name given, have that property with one of the given + * values, e.g. `Map("executionID" -> Set("a", "b"))` means `executionID IN (a, b)`; every property + * needs at least one value, otherwise an `IllegalArgumentException` is thrown + * @param from - only checkpoints with process start time at or after this time (inclusive) + * @param to - only checkpoints with process start time before this time (exclusive) + * @param latestFirst - order of the checkpoints by their process start time: latest first if `true`, earliest first if + * `false`, server default (latest first) if not specified */ case class CheckpointFilter( name: Option[String] = None, - properties: Map[String, Set[String]] = Map.empty + properties: Map[String, Set[String]] = Map.empty, + from: Option[ZonedDateTime] = None, + to: Option[ZonedDateTime] = None, + latestFirst: Option[Boolean] = None ) { require( @@ -47,7 +57,11 @@ case class CheckpointFilter( ) private[reader] def toQueryParams: Map[String, String] = { - name.map(QueryParamNames.CheckpointName -> _).toMap ++ propertiesParam + name.map(QueryParamNames.CheckpointName -> _).toMap ++ + propertiesParam ++ + from.map(QueryParamNames.From -> _.toInstant.toString) ++ + to.map(QueryParamNames.To -> _.toInstant.toString) ++ + latestFirst.map(QueryParamNames.LatestFirst -> _.toString) } private def propertiesParam: Option[(String, String)] = { diff --git a/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala b/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala index f1bf111f9..e17baddcd 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala @@ -22,4 +22,7 @@ object QueryParamNames { final val CheckpointName = "checkpoint-name" final val CheckpointProperties = "checkpoint-properties" final val IncludeProperties = "include-properties" + final val LatestFirst = "latest-first" + final val From = "from" + final val To = "to" } diff --git a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala index dc918b00d..e881c25c4 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala @@ -307,7 +307,7 @@ class FlowReaderUnitTests extends AnyFunSuiteLike { assert(result.isRight) } - test("The flow checkpoints are queried with name and multi-value properties in a single request") { + test("The flow checkpoints are queried with name, multi-value properties and time window in a single request") { implicit val server: SttpBackendStub[Identity, capabilities.WebSockets] = SttpBackendStub.synchronous .whenRequestMatchesPartial { case r if r.uri.path.endsWith(List("partitionings")) => @@ -320,12 +320,18 @@ class FlowReaderUnitTests extends AnyFunSuiteLike { assert(r.uri.querySegments.contains(KeyValue("include-properties", "true"))) assert(r.uri.querySegments.contains(KeyValue("checkpoint-name", "Test checkpoints 1"))) assert(r.uri.querySegments.contains(KeyValue("checkpoint-properties", multiValuePropertiesEncoded))) + assert(r.uri.querySegments.contains(KeyValue("from", "2026-06-01T00:00:00Z"))) + assert(r.uri.querySegments.contains(KeyValue("to", "2026-08-01T00:00:00Z"))) + assert(r.uri.querySegments.contains(KeyValue("latest-first", "false"))) Response.ok(checkpointsResponse) } val filter = CheckpointFilter( name = Some("Test checkpoints 1"), - properties = Map("executionID" -> Set("id2", "id1")) + properties = Map("executionID" -> Set("id2", "id1")), + from = Some(ZonedDateTime.parse("2026-06-01T02:00:00+02:00")), + to = Some(ZonedDateTime.parse("2026-08-01T00:00:00Z")), + latestFirst = Some(false) ) val result = FlowReader(AtumPartitions(List("a" -> "b", "c" -> "d"))) .getCheckpointsPage(pageSize = 5, offset = 10, includeProperties = true, filter = filter) diff --git a/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala index 2226e4c3e..9b7733d7f 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala @@ -277,7 +277,7 @@ class PartitioningReaderUnitTests extends AnyFunSuiteLike { assert(result.isRight) } - test("The partitioning checkpoints are queried with name and multi-value properties in a single request") { + test("The partitioning checkpoints are queried with name, multi-value properties and time window in a single request") { implicit val server: SttpBackendStub[Identity, capabilities.WebSockets] = SttpBackendStub.synchronous .whenRequestMatchesPartial { case r if r.uri.path.endsWith(List(V2Paths.Partitionings)) => @@ -288,12 +288,16 @@ class PartitioningReaderUnitTests extends AnyFunSuiteLike { assert(r.uri.querySegments.contains(KeyValue("checkpoint-name", "Test checkpoints 1"))) // base64url-encoded {"executionID":["id1","id2"]} assert(r.uri.querySegments.contains(KeyValue("checkpoint-properties", "eyJleGVjdXRpb25JRCI6WyJpZDEiLCJpZDIiXX0="))) + assert(r.uri.querySegments.contains(KeyValue("from", "2026-06-01T00:00:00Z"))) + assert(!r.uri.paramsMap.contains("to")) + assert(!r.uri.paramsMap.contains("latest-first")) Response.ok(checkpointsResponse) } val filter = CheckpointFilter( name = Some("Test checkpoints 1"), - properties = Map("executionID" -> Set("id1", "id2")) + properties = Map("executionID" -> Set("id1", "id2")), + from = Some(ZonedDateTime.parse("2026-06-01T00:00:00Z")) ) val result = PartitioningReader(AtumPartitions(List("a" -> "b", "c" -> "d"))).getCheckpointsPage(filter = filter) assert(result.map(_.data.size) == Right(2)) diff --git a/reader/src/test/scala/za/co/absa/atum/reader/requests/CheckpointFilterUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/requests/CheckpointFilterUnitTests.scala index 9c4ed8ee7..74ed24ae6 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/requests/CheckpointFilterUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/requests/CheckpointFilterUnitTests.scala @@ -19,6 +19,8 @@ package za.co.absa.atum.reader.requests import org.scalatest.funsuite.AnyFunSuiteLike import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ +import java.time.ZonedDateTime + class CheckpointFilterUnitTests extends AnyFunSuiteLike { test("An empty filter adds no query params") { @@ -43,13 +45,19 @@ class CheckpointFilterUnitTests extends AnyFunSuiteLike { ) } - test("Name and properties are combined") { - val filter = CheckpointFilter(name = Some("checkpoint name"), properties = Map("env" -> Set("prod"))) + test("The time window is sent as UTC instants and all filter parts are combined") { + val filter = CheckpointFilter( + name = Some("checkpoint name"), + from = Some(ZonedDateTime.parse("2026-06-01T02:00:00+02:00[Europe/Budapest]")), + to = Some(ZonedDateTime.parse("2026-08-01T00:00:00.5Z")), + latestFirst = Some(false) + ) assert( filter.toQueryParams == Map( "checkpoint-name" -> "checkpoint name", - // base64url-encoded {"env":"prod"} - "checkpoint-properties" -> "eyJlbnYiOiJwcm9kIn0=" + "from" -> "2026-06-01T00:00:00Z", + "to" -> "2026-08-01T00:00:00.500Z", + "latest-first" -> "false" ) ) } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala index 2534988f7..b04dc1f8b 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala @@ -35,6 +35,8 @@ import doobie.postgres.implicits._ import za.co.absa.atum.server.model.database.CheckpointItemWithPartitioningFromDB +import java.time.ZonedDateTime + class GetFlowCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieMultipleResultFunctionWithAggStatus[GetFlowCheckpointsArgs, Option[ CheckpointItemWithPartitioningFromDB @@ -44,7 +46,10 @@ class GetFlowCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task] fr"${input.limit}", fr"${input.offset}", fr"${input.checkpointName}", - fr"${input.checkpointProperties.map(_.asJson)}" + fr"${input.checkpointProperties.map(_.asJson)}", + fr"${input.latestFirst}", + fr"${input.from}", + fr"${input.to}" ) ) with StandardStatusHandling @@ -73,7 +78,10 @@ object GetFlowCheckpoints { limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, Seq[String]]] + checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime] ) val layer: URLayer[PostgresDatabaseProvider, GetFlowCheckpoints] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala index b43ec3662..0632ac744 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala @@ -35,6 +35,8 @@ import io.circe.generic.auto._ import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstRowStatusAggregator import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling +import java.time.ZonedDateTime + class GetPartitioningCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieMultipleResultFunctionWithAggStatus[GetPartitioningCheckpointsArgs, Option[ CheckpointItemFromDB.Paginated @@ -45,7 +47,9 @@ class GetPartitioningCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngi fr"${args.offset}", fr"${args.checkpointName}", fr"${args.checkpointProperties.map(_.asJson)}", - fr"${args.latestFirst}" + fr"${args.latestFirst}", + fr"${args.from}", + fr"${args.to}" ) ) with StandardStatusHandling @@ -72,7 +76,9 @@ object GetPartitioningCheckpoints { offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], - latestFirst: Option[Boolean] + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime] ) val layer: URLayer[PostgresDatabaseProvider, GetPartitioningCheckpoints] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointController.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointController.scala index 428f55a1e..4d521b999 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointController.scala @@ -22,6 +22,7 @@ import za.co.absa.atum.model.envelopes.SuccessResponse.{PaginatedResponse, Singl import zio.IO import zio.macros.accessible +import java.time.ZonedDateTime import java.util.UUID @accessible @@ -45,6 +46,8 @@ trait CheckpointController { checkpointName: Option[String] = None, checkpointProperties: Option[Map[String, Seq[String]]] = None, latestFirst: Option[Boolean] = None, + from: Option[ZonedDateTime] = None, + to: Option[ZonedDateTime] = None, includeProperties: Boolean ): IO[ErrorResponse, PaginatedResponse[CheckpointV2DTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerImpl.scala index 46897914e..464cf05ec 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerImpl.scala @@ -25,6 +25,7 @@ import za.co.absa.atum.server.api.v2.service.CheckpointService import za.co.absa.atum.server.model.PaginatedResult import zio._ +import java.time.ZonedDateTime import java.util.UUID class CheckpointControllerImpl(checkpointService: CheckpointService) extends CheckpointController with BaseController { @@ -66,6 +67,8 @@ class CheckpointControllerImpl(checkpointService: CheckpointService) extends Che checkpointName: Option[String] = None, checkpointProperties: Option[Map[String, Seq[String]]] = None, latestFirst: Option[Boolean] = None, + from: Option[ZonedDateTime] = None, + to: Option[ZonedDateTime] = None, includeProperties: Boolean ): IO[ErrorResponse, PaginatedResponse[CheckpointV2DTO]] = { mapToPaginatedResponse( @@ -79,6 +82,8 @@ class CheckpointControllerImpl(checkpointService: CheckpointService) extends Che checkpointName, checkpointProperties, latestFirst, + from, + to, includeProperties ) ) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowController.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowController.scala index 23be150c6..90cca4761 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowController.scala @@ -22,6 +22,8 @@ import za.co.absa.atum.model.envelopes.SuccessResponse.PaginatedResponse import zio.IO import zio.macros.accessible +import java.time.ZonedDateTime + @accessible trait FlowController { @@ -31,6 +33,9 @@ trait FlowController { offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ErrorResponse, PaginatedResponse[CheckpointWithPartitioningDTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerImpl.scala index 8a031b0f4..3d1419de0 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerImpl.scala @@ -24,6 +24,8 @@ import za.co.absa.atum.server.api.v2.service.FlowService import za.co.absa.atum.server.model.PaginatedResult import zio._ +import java.time.ZonedDateTime + class FlowControllerImpl(flowService: FlowService) extends FlowController with BaseController { // to be replaced (and moved to checkpointcontroller) with new implementation in #233 @@ -33,11 +35,24 @@ class FlowControllerImpl(flowService: FlowService) extends FlowController with B offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ErrorResponse, PaginatedResponse[CheckpointWithPartitioningDTO]] = { val flowData = serviceCall[PaginatedResult[CheckpointWithPartitioningDTO], PaginatedResult[CheckpointWithPartitioningDTO]]( - flowService.getFlowCheckpoints(flowId, limit, offset, checkpointName, checkpointProperties, includeProperties) + flowService.getFlowCheckpoints( + flowId, + limit, + offset, + checkpointName, + checkpointProperties, + latestFirst, + from, + to, + includeProperties + ) ) mapToPaginatedResponse(limit, offset, flowData) } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala index 719cb657d..34db55dec 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala @@ -20,7 +20,7 @@ import sttp.model.StatusCode import sttp.tapir.generic.auto.schemaForCaseClass import sttp.tapir.json.circe.jsonBody import sttp.tapir.ztapir._ -import sttp.tapir.{Codec, CodecFormat, DecodeResult, PublicEndpoint, ValidationResult, Validator} +import sttp.tapir.{Codec, CodecFormat, DecodeResult, EndpointInput, PublicEndpoint, ValidationResult, Validator} import za.co.absa.atum.model.ApiPaths._ import za.co.absa.atum.model.dto._ import za.co.absa.atum.model.envelopes.ErrorResponse @@ -29,6 +29,7 @@ import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ import za.co.absa.atum.server.api.v2.controller.{CheckpointController, FlowController, PartitioningController} import za.co.absa.atum.server.api.common.http.{BaseEndpoints, HttpEnv} +import java.time.ZonedDateTime import java.util.UUID import scala.util.{Failure, Success, Try} @@ -58,6 +59,21 @@ object Endpoints extends BaseEndpoints { ) }) + // Optional half-open window `from <= process start time < to` for checkpoint queries, supplied as ISO-8601 date-times + // with an offset, e.g. `from=2026-06-01T00:00:00Z`. A malformed value, or `from` not before `to`, results in a + // 400 Bad Request. + private val checkpointTimeWindowInput: EndpointInput[(Option[ZonedDateTime], Option[ZonedDateTime])] = + query[Option[ZonedDateTime]]("from") + .description("inclusive lower bound of the checkpoint's process start time (ISO-8601 date-time with offset)") + .and( + query[Option[ZonedDateTime]]("to") + .description("exclusive upper bound of the checkpoint's process start time (ISO-8601 date-time with offset)") + ) + .validate(Validator.custom { + case (Some(from), Some(to)) if !from.isBefore(to) => ValidationResult.Invalid("'from' must be before 'to'") + case _ => ValidationResult.Valid + }) + val postCheckpointEndpoint : PublicEndpoint[(Long, CheckpointV2DTO), ErrorResponse, (SingleSuccessResponse[CheckpointV2DTO], String), Any] = { apiV2.post @@ -126,7 +142,7 @@ object Endpoints extends BaseEndpoints { } val getPartitioningCheckpointsEndpoint - : PublicEndpoint[(Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Boolean), ErrorResponse, PaginatedResponse[ + : PublicEndpoint[(Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Option[ZonedDateTime], Option[ZonedDateTime], Boolean), ErrorResponse, PaginatedResponse[ CheckpointV2DTO ], Any] = { apiV2.get @@ -137,6 +153,7 @@ object Endpoints extends BaseEndpoints { .in(query[Option[String]]("checkpoint-name")) .in(query[Option[Map[String, Seq[String]]]]("checkpoint-properties")) .in(query[Option[Boolean]]("latest-first")) + .in(checkpointTimeWindowInput) .in(query[Boolean]("include-properties").default(false)) .out(statusCode(StatusCode.Ok)) .out(jsonBody[PaginatedResponse[CheckpointV2DTO]]) @@ -144,7 +161,7 @@ object Endpoints extends BaseEndpoints { } val getFlowCheckpointsEndpoint - : PublicEndpoint[(Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Boolean), ErrorResponse, PaginatedResponse[ + : PublicEndpoint[(Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Option[ZonedDateTime], Option[ZonedDateTime], Boolean), ErrorResponse, PaginatedResponse[ CheckpointWithPartitioningDTO ], Any] = { apiV2.get @@ -153,6 +170,8 @@ object Endpoints extends BaseEndpoints { .in(query[Long]("offset").default(0L).validate(Validator.min(0L))) .in(query[Option[String]]("checkpoint-name")) .in(query[Option[Map[String, Seq[String]]]]("checkpoint-properties")) + .in(query[Option[Boolean]]("latest-first")) + .in(checkpointTimeWindowInput) .in(query[Boolean]("include-properties").default(false)) .out(statusCode(StatusCode.Ok)) .out(jsonBody[PaginatedResponse[CheckpointWithPartitioningDTO]]) @@ -261,23 +280,23 @@ object Endpoints extends BaseEndpoints { } ), createServerEndpoint[ - (Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Boolean), + (Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Option[ZonedDateTime], Option[ZonedDateTime], Boolean), ErrorResponse, PaginatedResponse[CheckpointV2DTO] ]( getPartitioningCheckpointsEndpoint, - { case (partitioningId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], includeProperties: Boolean) => - CheckpointController.getPartitioningCheckpoints(partitioningId, limit, offset, checkpointName, checkpointProperties, latestFirst, includeProperties) + { case (partitioningId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], from: Option[ZonedDateTime], to: Option[ZonedDateTime], includeProperties: Boolean) => + CheckpointController.getPartitioningCheckpoints(partitioningId, limit, offset, checkpointName, checkpointProperties, latestFirst, from, to, includeProperties) } ), createServerEndpoint[ - (Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Boolean), + (Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Option[ZonedDateTime], Option[ZonedDateTime], Boolean), ErrorResponse, PaginatedResponse[CheckpointWithPartitioningDTO] ]( getFlowCheckpointsEndpoint, - { case (flowId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], includeProperties: Boolean) => - FlowController.getFlowCheckpoints(flowId, limit, offset, checkpointName, checkpointProperties, includeProperties) + { case (flowId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], from: Option[ZonedDateTime], to: Option[ZonedDateTime], includeProperties: Boolean) => + FlowController.getFlowCheckpoints(flowId, limit, offset, checkpointName, checkpointProperties, latestFirst, from, to, includeProperties) } ), createServerEndpoint(getPartitioningByIdEndpoint, PartitioningController.getPartitioningById), diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepository.scala index 14856ecf2..018cf3df6 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepository.scala @@ -22,6 +22,7 @@ import za.co.absa.atum.server.model.PaginatedResult import zio._ import zio.macros.accessible +import java.time.ZonedDateTime import java.util.UUID @accessible @@ -39,6 +40,8 @@ trait CheckpointRepository { checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[DatabaseError, PaginatedResult[CheckpointV2DTO]] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryImpl.scala index 3a8cb1c22..d1c13b425 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryImpl.scala @@ -31,6 +31,7 @@ import za.co.absa.atum.server.model.database.CheckpointItemFromDB import zio._ import zio.interop.catz.asyncInstance +import java.time.ZonedDateTime import java.util.UUID class CheckpointRepositoryImpl( @@ -77,11 +78,22 @@ class CheckpointRepositoryImpl( checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[DatabaseError, PaginatedResult[CheckpointV2DTO]] = { dbMultipleResultCallWithAggregatedStatus( getPartitioningCheckpointsFn( - GetPartitioningCheckpointsArgs(partitioningId, limit, offset, checkpointName, checkpointProperties, latestFirst) + GetPartitioningCheckpointsArgs( + partitioningId, + limit, + offset, + checkpointName, + checkpointProperties, + latestFirst, + from, + to + ) ), "getPartitioningCheckpoints" ) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepository.scala index d47df8ff1..15e9b9951 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepository.scala @@ -22,6 +22,8 @@ import za.co.absa.atum.server.model.PaginatedResult import zio._ import zio.macros.accessible +import java.time.ZonedDateTime + @accessible trait FlowRepository { @@ -31,6 +33,9 @@ trait FlowRepository { offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[DatabaseError, PaginatedResult[CheckpointWithPartitioningDTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryImpl.scala index b668384be..d861b046d 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryImpl.scala @@ -29,6 +29,8 @@ import za.co.absa.atum.server.model.database.CheckpointItemWithPartitioningFromD import zio._ import zio.interop.catz.asyncInstance +import java.time.ZonedDateTime + class FlowRepositoryImpl( getFlowCheckpointsFn: GetFlowCheckpoints, override val getCheckpointPropertiesFn: GetCheckpointProperties @@ -40,10 +42,15 @@ class FlowRepositoryImpl( offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[DatabaseError, PaginatedResult[CheckpointWithPartitioningDTO]] = { dbMultipleResultCallWithAggregatedStatus( - getFlowCheckpointsFn(GetFlowCheckpointsArgs(flowId, limit, offset, checkpointName, checkpointProperties)), + getFlowCheckpointsFn( + GetFlowCheckpointsArgs(flowId, limit, offset, checkpointName, checkpointProperties, latestFirst, from, to) + ), "getFlowCheckpoints" ) .map(_.flatten) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointService.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointService.scala index 4e6d4439e..c59ca56cd 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointService.scala @@ -22,6 +22,7 @@ import za.co.absa.atum.server.model.PaginatedResult import zio.IO import zio.macros.accessible +import java.time.ZonedDateTime import java.util.UUID @accessible @@ -42,6 +43,8 @@ trait CheckpointService { checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ServiceError, PaginatedResult[CheckpointV2DTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceImpl.scala index 05428e894..a53b57285 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceImpl.scala @@ -23,6 +23,7 @@ import za.co.absa.atum.server.api.v2.repository.CheckpointRepository import za.co.absa.atum.server.model.PaginatedResult import zio._ +import java.time.ZonedDateTime import java.util.UUID class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends CheckpointService with BaseService { @@ -52,6 +53,8 @@ class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ServiceError, PaginatedResult[CheckpointV2DTO]] = { repositoryCall( @@ -62,6 +65,8 @@ class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends checkpointName, checkpointProperties, latestFirst, + from, + to, includeProperties ), "getPartitioningCheckpoints" diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowService.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowService.scala index 27e0bbe5c..5563086b9 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowService.scala @@ -22,6 +22,8 @@ import za.co.absa.atum.server.model.PaginatedResult import zio._ import zio.macros.accessible +import java.time.ZonedDateTime + @accessible trait FlowService { @@ -31,6 +33,9 @@ trait FlowService { offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ServiceError, PaginatedResult[CheckpointWithPartitioningDTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowServiceImpl.scala index 936979a0c..555b23271 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowServiceImpl.scala @@ -23,6 +23,8 @@ import za.co.absa.atum.server.api.v2.repository.FlowRepository import za.co.absa.atum.server.model.PaginatedResult import zio._ +import java.time.ZonedDateTime + class FlowServiceImpl(flowRepository: FlowRepository) extends FlowService with BaseService { override def getFlowCheckpoints( @@ -31,10 +33,23 @@ class FlowServiceImpl(flowRepository: FlowRepository) extends FlowService with B offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ServiceError, PaginatedResult[CheckpointWithPartitioningDTO]] = { repositoryCall( - flowRepository.getFlowCheckpoints(flowId, limit, offset, checkpointName, checkpointProperties, includeProperties), + flowRepository.getFlowCheckpoints( + flowId, + limit, + offset, + checkpointName, + checkpointProperties, + latestFirst, + from, + to, + includeProperties + ), "getFlowCheckpoints" ) } diff --git a/server/src/main/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDB.scala index 25b1cf9e7..2ad4d5337 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDB.scala @@ -98,10 +98,8 @@ object CheckpointItemWithPartitioningFromDB { checkpointItems: Seq[CheckpointItemWithPartitioningFromDB] ): Either[Throwable, Seq[CheckpointWithPartitioningDTO]] = { val groupedItems = checkpointItems.groupBy(_.idCheckpoint) - val orderedCheckpointIds = checkpointItems - .sortBy(_.checkpointStartTime)(Ordering[ZonedDateTime].reverse) - .map(_.idCheckpoint) - .distinct + // the DB function returns the items already ordered (latest or earliest first, as requested), so keep that order + val orderedCheckpointIds = checkpointItems.map(_.idCheckpoint).distinct val result = orderedCheckpointIds.map { id => fromItemsToCheckpointWithPartitioningDTO(groupedItems(id)) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala index d21a68ece..fda0905e4 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala @@ -97,7 +97,10 @@ object GetFlowCheckpointsIntegrationTests extends ConfigProviderTest { limit = 10, offset = 0L, checkpointName = Some("TestCheckpointName"), - checkpointProperties = None + checkpointProperties = None, + latestFirst = None, + from = None, + to = None ) for { @@ -112,7 +115,10 @@ object GetFlowCheckpointsIntegrationTests extends ConfigProviderTest { limit = 2, offset = offset, checkpointName = Some(checkpointName), - checkpointProperties = Some(Map("executionID" -> Seq("a", "b"))) + checkpointProperties = Some(Map("executionID" -> Seq("a", "b"))), + latestFirst = None, + from = None, + to = None ) ) @@ -131,6 +137,30 @@ object GetFlowCheckpointsIntegrationTests extends ConfigProviderTest { firstPage == (Seq(matching3, matching2), true), secondPage == (Seq(matching1), false) ) + }, + test("Should apply the earliest-first order and the process start time window") { + def getWindowPage(flowId: Long) = getPage( + GetFlowCheckpointsArgs( + flowId = flowId, + limit = 10, + offset = 0L, + checkpointName = None, + checkpointProperties = None, + latestFirst = Some(false), + from = Some(firstStartTime.plusMinutes(2)), + to = Some(firstStartTime.plusMinutes(4)) + ) + ) + + for { + partitioningId <- createPartitioning + flowId <- getMainFlowId(partitioningId) + _ <- writeCheckpoint(partitioningId, checkpointName, None, startMinute = 1) + atWindowStart <- writeCheckpoint(partitioningId, checkpointName, None, startMinute = 2) + inWindow <- writeCheckpoint(partitioningId, checkpointName, None, startMinute = 3) + _ <- writeCheckpoint(partitioningId, checkpointName, None, startMinute = 4) + page <- getWindowPage(flowId) + } yield assertTrue(page == (Seq(atWindowStart, inWindow), false)) } ).provide( GetFlowCheckpoints.layer, diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowPartitioningsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowPartitioningsIntegrationTests.scala index af3a3232c..11bed2633 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowPartitioningsIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowPartitioningsIntegrationTests.scala @@ -16,18 +16,37 @@ package za.co.absa.atum.server.api.database.flows.functions +import za.co.absa.atum.model.dto.{PartitionDTO, PartitioningSubmitV2DTO} import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.flows.functions.GetFlowPartitionings.GetFlowPartitioningsArgs +import za.co.absa.atum.server.api.database.runs.functions.{CreatePartitioning, GetPartitioningMainFlow} import za.co.absa.db.fadb.exceptions.DataNotFoundException import za.co.absa.db.fadb.status.FunctionStatus import zio.{Scope, ZIO} import zio.test.{Spec, TestEnvironment, assertTrue} import zio.interop.catz.asyncInstance +import java.util.UUID + object GetFlowPartitioningsIntegrationTests extends ConfigProviderTest { + private def createPartitioning(parentId: Option[Long]): ZIO[CreatePartitioning, Throwable, Long] = + ZIO + .serviceWithZIO[CreatePartitioning]( + _(PartitioningSubmitV2DTO(Seq(PartitionDTO("page", UUID.randomUUID().toString)), parentId, "Pagination author")) + ) + .flatMap(ZIO.fromEither(_)) + .map(_.data) + + private def getPage(flowId: Long, offset: Long): ZIO[GetFlowPartitionings, Throwable, Seq[(Long, Boolean)]] = + ZIO + .serviceWithZIO[GetFlowPartitionings](_(GetFlowPartitioningsArgs(flowId, 2, offset))) + .flatMap(ZIO.fromEither(_)) + .flatMap(rows => ZIO.foreach(rows)(row => ZIO.getOrFail(row.data))) + .map(_.map(row => (row.id, row.hasMore))) + override def spec: Spec[Unit with TestEnvironment with Scope, Any] = { suite("GetFlowPartitioningsIntegrationTests")( test("Returns expected DataNotFoundException when flow not found") { @@ -35,12 +54,34 @@ object GetFlowPartitioningsIntegrationTests extends ConfigProviderTest { getFlowPartitionings <- ZIO.service[GetFlowPartitionings] result <- getFlowPartitionings(GetFlowPartitioningsArgs(0L, 1, 1L)) } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(41, "Flow not found")))) + }, + test("Decodes ordered partitionings and paging metadata at positive offsets") { + for { + parentId <- createPartitioning(None) + childIds <- ZIO.foreach(1 to 4)(_ => createPartitioning(Some(parentId))) + ids = (parentId +: childIds).sorted + mainFlow <- ZIO.serviceWithZIO[GetPartitioningMainFlow](_(parentId)).flatMap(ZIO.fromEither(_)) + flowId <- ZIO.getOrFail(mainFlow.data.map(_.id)) + firstPage <- getPage(flowId, 0L) + middlePage <- getPage(flowId, 2L) + lastPartialPage <- getPage(flowId, 4L) + lastFullPage <- getPage(flowId, 3L) + beyondEndPage <- getPage(flowId, 6L) + } yield assertTrue( + firstPage == ids.take(2).map(id => (id, true)), + middlePage == ids.slice(2, 4).map(id => (id, true)), + lastPartialPage == Seq((ids.last, false)), + lastFullPage == ids.takeRight(2).map(id => (id, false)), + beyondEndPage.isEmpty + ) } ) }.provide( GetFlowPartitionings.layer, + CreatePartitioning.layer, + GetPartitioningMainFlow.layer, PostgresDatabaseProvider.layer, - TestTransactorProvider.layerWithRollback + TestTransactorProvider.layerWithSingleTransactionRollback ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala index 93fb53dbe..db1d04999 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala @@ -87,7 +87,9 @@ object GetPartitioningCheckpointsIntegrationTests extends ConfigProviderTest { offset = 0L, checkpointName = None, checkpointProperties = None, - latestFirst = None + latestFirst = None, + from = None, + to = None ) for { @@ -103,7 +105,10 @@ object GetPartitioningCheckpointsIntegrationTests extends ConfigProviderTest { offset = offset, checkpointName = Some(checkpointName), checkpointProperties = Some(Map("executionID" -> Seq("a", "b"))), - latestFirst = latestFirst + latestFirst = latestFirst, + from = None, + to = None + ) ) @@ -123,6 +128,29 @@ object GetPartitioningCheckpointsIntegrationTests extends ConfigProviderTest { secondPage == (Seq(matching1), false), earliestFirstPage == (Seq(matching1, matching2), true) ) + }, + test("Should apply the earliest-first order and the process start time window") { + def getWindowPage(partitioningId: Long) = getPage( + GetPartitioningCheckpointsArgs( + partitioningId = partitioningId, + limit = 10, + offset = 0L, + checkpointName = None, + checkpointProperties = None, + latestFirst = Some(false), + from = Some(firstStartTime.plusMinutes(2)), + to = Some(firstStartTime.plusMinutes(4)) + ) + ) + + for { + partitioningId <- createPartitioning + _ <- writeCheckpoint(partitioningId, checkpointName, None, startMinute = 1) + atWindowStart <- writeCheckpoint(partitioningId, checkpointName, None, startMinute = 2) + inWindow <- writeCheckpoint(partitioningId, checkpointName, None, startMinute = 3) + _ <- writeCheckpoint(partitioningId, checkpointName, None, startMinute = 4) + page <- getWindowPage(partitioningId) + } yield assertTrue(page == (Seq(atWindowStart, inWindow), false)) } ).provide( GetPartitioningCheckpoints.layer, diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerUnitTests.scala index 9a846d9d5..b44674f70 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerUnitTests.scala @@ -51,11 +51,11 @@ object CheckpointControllerUnitTests extends ConfigProviderTest with TestData { when(checkpointServiceMock.getCheckpoint(partitioningId1, checkpointV2DTO3.id, includeProperties = false)) .thenReturn(ZIO.fail(GeneralServiceError("boom!"))) - when(checkpointServiceMock.getPartitioningCheckpoints(partitioningId1, 10, 0L, None, None, None, includeProperties = false)) + when(checkpointServiceMock.getPartitioningCheckpoints(partitioningId1, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultHasMore(Seq(checkpointV2DTO1)))) - when(checkpointServiceMock.getPartitioningCheckpoints(partitioningId2, 10, 0L, None, None, None, includeProperties = false)) + when(checkpointServiceMock.getPartitioningCheckpoints(partitioningId2, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultNoMore(Seq(checkpointV2DTO1)))) - when(checkpointServiceMock.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, includeProperties = false)) + when(checkpointServiceMock.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundServiceError("Partitioning not found"))) private val checkpointServiceMockLayer = ZLayer.succeed(checkpointServiceMock) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerUnitTests.scala index d78be581e..eabe38774 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerUnitTests.scala @@ -29,11 +29,11 @@ import zio.test._ object FlowControllerUnitTests extends ZIOSpecDefault with TestData { private val flowServiceMock = mock(classOf[FlowService]) - when(flowServiceMock.getFlowCheckpoints(1L, 5, 2L, None, None, includeProperties = false)) + when(flowServiceMock.getFlowCheckpoints(1L, 5, 2L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultHasMore(Seq(checkpointWithPartitioningDTO1)))) - when(flowServiceMock.getFlowCheckpoints(2L, 5, 0L, None, None, includeProperties = false)) + when(flowServiceMock.getFlowCheckpoints(2L, 5, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultNoMore(Seq(checkpointWithPartitioningDTO2)))) - when(flowServiceMock.getFlowCheckpoints(3L, 5, 0L, None, None, includeProperties = false)) + when(flowServiceMock.getFlowCheckpoints(3L, 5, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundServiceError("Flow not found"))) private val flowServiceMockLayer = ZLayer.succeed(flowServiceMock) @@ -43,20 +43,20 @@ object FlowControllerUnitTests extends ZIOSpecDefault with TestData { suite("GetFlowCheckpointsV2Suite")( test("Returns expected Seq[CheckpointV2DTO] with Pagination indicating there is more data available") { for { - result <- FlowController.getFlowCheckpoints(1L, 5, 2L, None, None, includeProperties = false) + result <- FlowController.getFlowCheckpoints(1L, 5, 2L, None, None, None, None, None, includeProperties = false) } yield assertTrue( result.data == Seq(checkpointWithPartitioningDTO1) && result.pagination == Pagination(5, 2, hasMore = true) ) }, test("Returns expected Seq[CheckpointV2DTO] with Pagination indicating there is no more data available") { for { - result <- FlowController.getFlowCheckpoints(2L, 5, 0L, None, None, includeProperties = false) + result <- FlowController.getFlowCheckpoints(2L, 5, 0L, None, None, None, None, None, includeProperties = false) } yield assertTrue( result.data == Seq(checkpointWithPartitioningDTO2) && result.pagination == Pagination(5, 0, hasMore = false) ) }, test("Returns expected NotFoundServiceError when service returns NotFoundServiceError") { - assertZIO(FlowController.getFlowCheckpoints(3L, 5, 0L, None, None, includeProperties = false).exit)( + assertZIO(FlowController.getFlowCheckpoints(3L, 5, 0L, None, None, None, None, None, includeProperties = false).exit)( failsWithA[NotFoundErrorResponse] ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala index ef56c4d42..fc5ba448e 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala @@ -33,6 +33,7 @@ import zio.test.Assertion.equalTo import zio.test.{Spec, TestEnvironment, ZIOSpecDefault, assertZIO} import zio.{Scope, ZIO, ZLayer} +import java.time.ZonedDateTime import java.util.UUID object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData { @@ -46,11 +47,11 @@ object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData private val encodedExecutionIdProperties = "eyJleGVjdXRpb25JRCI6IjAxOWY4OTgxLTc4NjgtNzlmYy04MWQzLTgxNDNhNDcwNmY4YSJ9" - when(flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, None, includeProperties = false)) + when(flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn( ZIO.succeed(PaginatedResponse(Seq(checkpointWithPartitioningDTO1), Pagination(5, 0, hasMore = true), uuid)) ) - when(flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, None, includeProperties = true)) + when(flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, None, None, None, None, includeProperties = true)) .thenReturn( ZIO.succeed( PaginatedResponse( @@ -60,24 +61,56 @@ object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData ) ) ) - when(flowControllerMockV2.getFlowCheckpoints(2L, 5, 0L, None, None, includeProperties = false)) + when(flowControllerMockV2.getFlowCheckpoints(2L, 5, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn( ZIO.succeed(PaginatedResponse(Seq(checkpointWithPartitioningDTO2), Pagination(5, 0, hasMore = false), uuid)) ) - when(flowControllerMockV2.getFlowCheckpoints(3L, 5, 0L, None, None, includeProperties = false)) + when(flowControllerMockV2.getFlowCheckpoints(3L, 5, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundErrorResponse("Flow not found for a given ID"))) when( - flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, Some(executionIdProperties.view.mapValues(Seq(_)).toMap), includeProperties = false) + flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, Some(executionIdProperties.view.mapValues(Seq(_)).toMap), None, None, None, includeProperties = false) ) .thenReturn( ZIO.succeed(PaginatedResponse(Seq(checkpointWithPartitioningDTO1), Pagination(5, 0, hasMore = true), uuid)) ) + private val windowFrom = ZonedDateTime.parse("2026-06-01T00:00:00Z") + private val windowTo = ZonedDateTime.parse("2026-08-01T00:00:00Z") + + when( + flowControllerMockV2.getFlowCheckpoints( + 1L, 5, 0L, None, None, Some(false), Some(windowFrom), Some(windowTo), includeProperties = false + ) + ) + .thenReturn( + ZIO.succeed(PaginatedResponse(Seq(checkpointWithPartitioningDTO2), Pagination(5, 0, hasMore = false), uuid)) + ) + private val flowControllerMockLayerV2 = ZLayer.succeed(flowControllerMockV2) private val getFlowCheckpointServerEndpoint = Endpoints.getFlowCheckpointsEndpoint.zServerLogic({ - case (flowId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], includeProperties: Boolean) => - FlowController.getFlowCheckpoints(flowId, limit, offset, checkpointName, checkpointProperties, includeProperties) + case ( + flowId: Long, + limit: Int, + offset: Long, + checkpointName: Option[String], + checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], + includeProperties: Boolean + ) => + FlowController.getFlowCheckpoints( + flowId, + limit, + offset, + checkpointName, + checkpointProperties, + latestFirst, + from, + to, + includeProperties + ) }) def spec: Spec[TestEnvironment with Scope, Any] = { @@ -175,6 +208,47 @@ object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData ) ) }, + test("Returns an expected PaginatedResponse[CheckpointWithPartitioningDTO] when filtering by a time window, earliest first") { + val baseUri = + uri"https://test.com/api/v2/flows/1/checkpoints?limit=5&offset=0&latest-first=false&from=2026-06-01T00:00:00Z&to=2026-08-01T00:00:00Z" + val response = basicRequest + .get(baseUri) + .response(asJson[PaginatedResponse[CheckpointWithPartitioningDTO]]) + .send(backendStub) + + val body = response.map(_.body) + val statusCode = response.map(_.code) + + assertZIO(body <&> statusCode)( + equalTo( + Right(PaginatedResponse(Seq(checkpointWithPartitioningDTO2), Pagination(5, 0, hasMore = false), uuid)), + StatusCode.Ok + ) + ) + }, + test("Returns expected 400 when 'to' is not a valid date-time") { + val baseUri = uri"https://test.com/api/v2/flows/1/checkpoints?limit=5&offset=0&to=not-a-date-time" + val response = basicRequest + .get(baseUri) + .response(asJson[PaginatedResponse[CheckpointWithPartitioningDTO]]) + .send(backendStub) + + val statusCode = response.map(_.code) + + assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) + }, + test("Returns expected 400 when 'from' is after 'to'") { + val baseUri = + uri"https://test.com/api/v2/flows/1/checkpoints?limit=5&offset=0&from=2026-08-01T00:00:00Z&to=2026-06-01T00:00:00Z" + val response = basicRequest + .get(baseUri) + .response(asJson[PaginatedResponse[CheckpointWithPartitioningDTO]]) + .send(backendStub) + + val statusCode = response.map(_.code) + + assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) + }, test("Returns expected 400 when checkpoint-properties is not valid base64") { val baseUri = uri"https://test.com/api/v2/flows/1/checkpoints?limit=5&offset=0&checkpoint-properties=!!!not-base64!!!" diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala index 41e320fb9..630089907 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala @@ -33,6 +33,7 @@ import zio.test.Assertion.equalTo import zio.test.{Spec, TestEnvironment, ZIOSpecDefault, assertZIO} import zio.{Scope, ZIO, ZLayer} +import java.time.ZonedDateTime import java.util.UUID object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData { @@ -41,9 +42,12 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T private val uuid = UUID.randomUUID() - when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, includeProperties = false)) + private val windowFrom = ZonedDateTime.parse("2026-06-01T00:00:00Z") + private val windowTo = ZonedDateTime.parse("2026-08-01T00:00:00Z") + + when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(PaginatedResponse(Seq(checkpointV2DTO1), Pagination(10, 0, hasMore = true), uuid))) - when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, includeProperties = true)) + when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, None, None, includeProperties = true)) .thenReturn( ZIO.succeed( PaginatedResponse( @@ -53,14 +57,20 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T ) ) ) - when(checkpointControllerMock.getPartitioningCheckpoints(1L, 20, 0L, None, None, None, includeProperties = false)) + when(checkpointControllerMock.getPartitioningCheckpoints(1L, 20, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(PaginatedResponse(Seq(checkpointV2DTO1), Pagination(20, 0, hasMore = false), uuid))) - when(checkpointControllerMock.getPartitioningCheckpoints(2L, 10, 0L, None, None, None, includeProperties = false)) + when(checkpointControllerMock.getPartitioningCheckpoints(2L, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundErrorResponse("partitioning not found"))) - when(checkpointControllerMock.getPartitioningCheckpoints(3L, 10, 0L, None, None, None, includeProperties = false)) + when(checkpointControllerMock.getPartitioningCheckpoints(3L, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(PaginatedResponse(Seq(checkpointV2DTO1), Pagination(10, 0, hasMore = true), uuid))) - when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, Some(Map("executionID" -> Seq("id1", "id2"))), None, includeProperties = false)) + when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, Some(Map("executionID" -> Seq("id1", "id2"))), None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(PaginatedResponse(Seq(checkpointV2DTO1), Pagination(10, 0, hasMore = true), uuid))) + when( + checkpointControllerMock.getPartitioningCheckpoints( + 1L, 10, 0L, None, None, None, Some(windowFrom), Some(windowTo), includeProperties = false + ) + ) + .thenReturn(ZIO.succeed(PaginatedResponse(Seq(checkpointV2DTO2), Pagination(10, 0, hasMore = false), uuid))) private val checkpointControllerMockLayer = ZLayer.succeed(checkpointControllerMock) @@ -74,6 +84,8 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ) => CheckpointController.getPartitioningCheckpoints( @@ -83,6 +95,8 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T checkpointName, checkpointProperties, latestFirst, + from, + to, includeProperties ) }) @@ -115,6 +129,52 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T ) ) }, + test("Returns an expected PaginatedResponse[CheckpointV2DTO] when filtering by a process start time window") { + val request = basicRequest + .get( + uri"https://test.com/api/v2/partitionings/1/checkpoints?limit=10&offset=0&from=2026-06-01T00:00:00Z&to=2026-08-01T00:00:00Z" + ) + .response(asJson[PaginatedResponse[CheckpointV2DTO]]) + + val response = request + .send(backendStub) + + val body = response.map(_.body) + val statusCode = response.map(_.code) + + assertZIO(body <&> statusCode)( + equalTo( + Right(PaginatedResponse(Seq(checkpointV2DTO2), Pagination(10, 0, hasMore = false), uuid)), + StatusCode.Ok + ) + ) + }, + test("Returns expected 400 when 'from' is not a valid date-time") { + val request = basicRequest + .get(uri"https://test.com/api/v2/partitionings/1/checkpoints?limit=10&offset=0&from=2026-06-01") + .response(asJson[PaginatedResponse[CheckpointV2DTO]]) + + val response = request + .send(backendStub) + + val statusCode = response.map(_.code) + + assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) + }, + test("Returns expected 400 when 'from' is not before 'to'") { + val request = basicRequest + .get( + uri"https://test.com/api/v2/partitionings/1/checkpoints?limit=10&offset=0&from=2026-08-01T00:00:00Z&to=2026-08-01T00:00:00Z" + ) + .response(asJson[PaginatedResponse[CheckpointV2DTO]]) + + val response = request + .send(backendStub) + + val statusCode = response.map(_.code) + + assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) + }, test("Returns an expected PaginatedResponse[CheckpointV2DTO] with more data available") { val request = basicRequest .get(uri"https://test.com/api/v2/partitionings/1/checkpoints?limit=10&offset=0") diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryUnitTests.scala index af5308870..ef54fb94e 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryUnitTests.scala @@ -62,11 +62,11 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { when(getCheckpointMockV2.apply(GetPartitioningCheckpointV2Args(partitioningId, checkpointV2DTO3.id))) .thenReturn(ZIO.fail(new Exception("boom!"))) - when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(0L, 10, 0L, None, None, None))) + when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(0L, 10, 0L, None, None, None, None, None))) .thenReturn(ZIO.right(Seq(Row(FunctionStatus(11, "ok"), Some(checkpointItemPaginatedFromDB))))) - when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(1L, 10, 0L, None, None, None))) + when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(1L, 10, 0L, None, None, None, None, None))) .thenReturn(ZIO.right(Seq(Row(FunctionStatus(11, "ok"), None)))) - when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(3L, 10, 0L, None, None, None))) + when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(3L, 10, 0L, None, None, None, None, None))) .thenReturn(ZIO.left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) when(getCheckpointPropertiesMock.apply(GetCheckpointPropertiesArgs(checkpointV2DTO1.id))) @@ -137,14 +137,14 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { suite("GetPartitioningCheckpointsSuite")( test("Returns expected Seq without properties") { for { - result <- CheckpointRepository.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, includeProperties = false) + result <- CheckpointRepository.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, None, None, includeProperties = false) } yield assertTrue( result.isInstanceOf[ResultHasMore[CheckpointV2DTO]] && result.data == Seq(checkpointV2DTO2) ) }, test("Returns expected Seq with properties") { for { - result <- CheckpointRepository.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, includeProperties = true) + result <- CheckpointRepository.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, None, None, includeProperties = true) } yield assertTrue( result.isInstanceOf[ResultHasMore[CheckpointV2DTO]] && result.data == Seq( checkpointV2DTO2.copy(properties = Some(Map("propName1" -> "propValue1"))) @@ -153,13 +153,13 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { }, test("Returns expected Seq.empty") { for { - result <- CheckpointRepository.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, includeProperties = false) + result <- CheckpointRepository.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, None, None, includeProperties = false) } yield assertTrue( result.isInstanceOf[ResultNoMore[CheckpointV2DTO]] && result.data == Seq.empty[CheckpointV2DTO] ) }, test("Returns expected NotFoundDatabaseError") { - assertZIO(CheckpointRepository.getPartitioningCheckpoints(3L, 10, 0L, None, None, None, includeProperties = false).exit)( + assertZIO(CheckpointRepository.getPartitioningCheckpoints(3L, 10, 0L, None, None, None, None, None, includeProperties = false).exit)( failsWithA[NotFoundDatabaseError] ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryUnitTests.scala index ff30f4b86..12c594385 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryUnitTests.scala @@ -38,7 +38,7 @@ object FlowRepositoryUnitTests extends ZIOSpecDefault with TestData { private val getFlowCheckpointsV2Mock = mock(classOf[GetFlowCheckpoints]) private val getCheckpointPropertiesMock: GetCheckpointProperties = mock(classOf[GetCheckpointProperties]) - when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(1, 1, 1L, None, None))) + when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(1, 1, 1L, None, None, None, None, None))) .thenReturn( ZIO.right( Seq( @@ -46,9 +46,9 @@ object FlowRepositoryUnitTests extends ZIOSpecDefault with TestData { ) ) ) - when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(2, 1, 1L, None, None))) + when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(2, 1, 1L, None, None, None, None, None))) .thenReturn(ZIO.right(Seq(Row(FunctionStatus(11, "success"), None)))) - when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(3, 1, 1L, None, None))) + when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(3, 1, 1L, None, None, None, None, None))) .thenReturn(ZIO.fail(DataNotFoundException(FunctionStatus(42, "Flow not found")))) when( @@ -67,12 +67,12 @@ object FlowRepositoryUnitTests extends ZIOSpecDefault with TestData { suite("GetFlowCheckpointsV2Suite")( test("Returns expected Right with CheckpointV2DTO without properties") { for { - result <- FlowRepository.getFlowCheckpoints(1, 1, 1L, None, None, includeProperties = false) + result <- FlowRepository.getFlowCheckpoints(1, 1, 1L, None, None, None, None, None, includeProperties = false) } yield assertTrue(result == ResultHasMore(Seq(checkpointWithPartitioningDTO1))) }, test("Returns expected Right with CheckpointV2DTO with properties") { for { - result <- FlowRepository.getFlowCheckpoints(1, 1, 1L, None, None, includeProperties = true) + result <- FlowRepository.getFlowCheckpoints(1, 1, 1L, None, None, None, None, None, includeProperties = true) } yield assertTrue( result == ResultHasMore( Seq(checkpointWithPartitioningDTO1.copy(properties = Some(Map("propName1" -> "propValue1")))) @@ -81,11 +81,11 @@ object FlowRepositoryUnitTests extends ZIOSpecDefault with TestData { }, test("Returns expected Right with CheckpointV2DTO") { for { - result <- FlowRepository.getFlowCheckpoints(2, 1, 1L, None, None, includeProperties = false) + result <- FlowRepository.getFlowCheckpoints(2, 1, 1L, None, None, None, None, None, includeProperties = false) } yield assertTrue(result == ResultNoMore(Seq.empty[CheckpointWithPartitioningDTO])) }, test("Returns expected DatabaseError") { - assertZIO(FlowRepository.getFlowCheckpoints(3, 1, 1L, None, None, includeProperties = false).exit)( + assertZIO(FlowRepository.getFlowCheckpoints(3, 1, 1L, None, None, None, None, None, includeProperties = false).exit)( failsWithA[NotFoundDatabaseError] ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceUnitTests.scala index e2c5406dc..eaba19979 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceUnitTests.scala @@ -45,9 +45,9 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { when(checkpointRepositoryMock.getCheckpoint(partitioningId, checkpointV2DTO2.id, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundDatabaseError("not found"))) - when(checkpointRepositoryMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, includeProperties = false)) + when(checkpointRepositoryMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultHasMore(Seq(checkpointV2DTO1)))) - when(checkpointRepositoryMock.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, includeProperties = false)) + when(checkpointRepositoryMock.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundDatabaseError("Partitioning not found"))) private val checkpointRepositoryMockLayer = ZLayer.succeed(checkpointRepositoryMock) @@ -92,13 +92,13 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { suite("GetPartitioningCheckpointsSuite")( test("Returns expected Right with Seq[CheckpointDTO]") { for { - result <- CheckpointService.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, includeProperties = false) + result <- CheckpointService.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, None, None, includeProperties = false) } yield assertTrue { result == ResultHasMore(Seq(checkpointV2DTO1)) } }, test("Returns expected NotFoundServiceError") { - assertZIO(CheckpointService.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, includeProperties = false).exit)( + assertZIO(CheckpointService.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, None, None, includeProperties = false).exit)( failsWithA[NotFoundServiceError] ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/service/FlowServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/service/FlowServiceUnitTests.scala index bcd867305..51233afbe 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/service/FlowServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/service/FlowServiceUnitTests.scala @@ -29,9 +29,9 @@ import zio.test._ object FlowServiceUnitTests extends ZIOSpecDefault with TestData { private val flowRepositoryMock = mock(classOf[FlowRepository]) - when(flowRepositoryMock.getFlowCheckpoints(1L, 1, 1L, None, None, includeProperties = false)) + when(flowRepositoryMock.getFlowCheckpoints(1L, 1, 1L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultHasMore(Seq(checkpointWithPartitioningDTO1)))) - when(flowRepositoryMock.getFlowCheckpoints(2L, 1, 1L, None, None, includeProperties = false)) + when(flowRepositoryMock.getFlowCheckpoints(2L, 1, 1L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundDatabaseError("Flow not found"))) private val flowRepositoryMockLayer = ZLayer.succeed(flowRepositoryMock) @@ -42,13 +42,13 @@ object FlowServiceUnitTests extends ZIOSpecDefault with TestData { suite("GetFlowCheckpointsV2Suite")( test("Returns expected PaginatedResult[CheckpointV2DTO]") { for { - result <- FlowService.getFlowCheckpoints(1L, 1, 1L, None, None, includeProperties = false) + result <- FlowService.getFlowCheckpoints(1L, 1, 1L, None, None, None, None, None, includeProperties = false) } yield assertTrue { result == ResultHasMore(Seq(checkpointWithPartitioningDTO1)) } }, test("Returns expected ServiceError") { - assertZIO(FlowService.getFlowCheckpoints(2L, 1, 1L, None, None, includeProperties = false).exit)( + assertZIO(FlowService.getFlowCheckpoints(2L, 1, 1L, None, None, None, None, None, includeProperties = false).exit)( failsWithA[NotFoundServiceError] ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDBUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDBUnitTests.scala index 87d2b75c6..1be1b3625 100644 --- a/server/src/test/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDBUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDBUnitTests.scala @@ -24,7 +24,7 @@ import java.util.UUID class CheckpointItemWithPartitioningFromDBUnitTests extends AnyFunSuiteLike with TestData { - test("groupAndConvertItemsToCheckpointWithPartitioningDTOs should group by idCheckpoint and sort by checkpointStartTime in descending order") { + test("groupAndConvertItemsToCheckpointWithPartitioningDTOs should group by idCheckpoint and keep the order of the DB rows") { val checkpointItem1 = CheckpointItemWithPartitioningFromDB( idCheckpoint = UUID.randomUUID(), checkpointName = "checkpoint1", @@ -57,14 +57,20 @@ class CheckpointItemWithPartitioningFromDBUnitTests extends AnyFunSuiteLike with hasMore = true ) - val checkpointItems = Seq(checkpointItem1, checkpointItem2) + // a second measure of the first checkpoint, i.e. another DB row of the same checkpoint + val checkpointItem1OtherMeasure = checkpointItem1.copy(measureName = "measure3", measuredColumns = Seq("col3")) - val result = CheckpointItemWithPartitioningFromDB.groupAndConvertItemsToCheckpointWithPartitioningDTOs(checkpointItems) + // earliest first, as returned by the DB for latest-first = false + val earliestFirst = CheckpointItemWithPartitioningFromDB + .groupAndConvertItemsToCheckpointWithPartitioningDTOs(Seq(checkpointItem1, checkpointItem1OtherMeasure, checkpointItem2)) + assert(earliestFirst.isRight) + assert(earliestFirst.toOption.get.map(_.id) == Seq(checkpointItem1.idCheckpoint, checkpointItem2.idCheckpoint)) + assert(earliestFirst.toOption.get.head.measurements.size == 2) - assert(result.isRight) - val checkpoints = result.toOption.get - - assert(checkpoints.size == 2) - assert(checkpoints.head.processStartTime.isAfter(checkpoints(1).processStartTime)) + // latest first, as returned by the DB by default + val latestFirst = CheckpointItemWithPartitioningFromDB + .groupAndConvertItemsToCheckpointWithPartitioningDTOs(Seq(checkpointItem2, checkpointItem1, checkpointItem1OtherMeasure)) + assert(latestFirst.isRight) + assert(latestFirst.toOption.get.map(_.id) == Seq(checkpointItem2.idCheckpoint, checkpointItem1.idCheckpoint)) } }