diff --git a/README.md b/README.md index 8e9880fb2..228a57284 100644 --- a/README.md +++ b/README.md @@ -108,12 +108,13 @@ drop support for Spark 2.4. If you need the agent to work with Spark 2.4 follow * Build these two modules and use them in your project ### Reader `reader/` -**NB!** -_This module is not yet implemented to an operational abilities and therefore not yet released._ +This module is a Scala client for applications reading metrics stored by the _Atum Service_. `FlowReader` and +`PartitioningReader` resolve partitionings and retrieve checkpoint pages through the server's REST API. +Applications control pagination; the Reader does not discover datasets from partial business keys or reconstruct +the original parent-child lineage graph. -This module is intended to be used whenever an application wants to read the metrics stored by the _Atum Service_. It -offers classes and methods to read the metrics from the database shielding away the complexity of accessing the _Atum Server_ -directly. +Reader features require a compatible server version. In particular, older servers can ignore checkpoint time-window +parameters; using a newer Reader alone does not enable server-side filtering. ### Server `server/` An API under construction that communicates with the Agent and with the persistent storage. It also provides measure diff --git a/agent/README.md b/agent/README.md index 70445690c..a85705f06 100644 --- a/agent/README.md +++ b/agent/README.md @@ -25,7 +25,7 @@ Use `AtumPartitions` to get an `AtumContext` from the service using the `AtumAge import za.co.absa.atum.agent.AtumAgent import za.co.absa.atum.model.types.basic.AtumPartitions -val atumPartitions = AtumPartitions("country" -> "za", "dataset" -> "customers") +val atumPartitions = AtumPartitions(List("country" -> "za", "dataset" -> "customers")) val atumContext1 = AtumAgent.getOrCreateAtumContext(atumPartitions) ``` @@ -45,7 +45,7 @@ val config = ConfigFactory.parseString( ) val customAgent = AtumAgent.fromConfig(config) -val atumPartitions = AtumPartitions("country" -> "za", "dataset" -> "customers") +val atumPartitions = AtumPartitions(List("country" -> "za", "dataset" -> "customers")) val atumContext = customAgent.getOrCreateAtumContext(atumPartitions) ``` @@ -53,11 +53,18 @@ val atumContext = customAgent.getOrCreateAtumContext(atumPartitions) A list of key values that maintains the order of arrival of the items, the `AtumService` is able to deliver the correct `AtumContext` according to the `AtumPartitions` we give it. ```scala - val atumPartitions = AtumPartitions().withPartitions(ListMap("name" -> "partition-name", "country" -> "SA", "gender" -> "female" )) +val atumPartitions = AtumPartitions(List( + "name" -> "partition-name", + "country" -> "SA", + "gender" -> "female" +)) - val subPartition = atumPartitions.addPartition("otherKey", "otherValue") +val withOtherKey = atumPartitions ++ AtumPartitions("otherKey" -> "otherValue") ``` +Constructing these keys does not register a parent-child relationship. Use +`AtumContext.subPartitionContext` to create a linked child context. + Control measures can also be overwritten, added or removed. ```scala diff --git a/database/src/main/postgres/flows/V0.8.0.1__get_flow_checkpoints.sql b/database/src/main/postgres/flows/V0.8.0.1__get_flow_checkpoints.sql new file mode 100644 index 000000000..4ce2a4e65 --- /dev/null +++ b/database/src/main/postgres/flows/V0.8.0.1__get_flow_checkpoints.sql @@ -0,0 +1,221 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- The previous overload is dropped so that the new signature (JSONB multi-value checkpoint properties filter and +-- the process start time window) is unambiguous when the function is called with named or defaulted arguments. +DROP FUNCTION IF EXISTS flows.get_flow_checkpoints(BIGINT, INT, BIGINT, TEXT, HSTORE, BOOLEAN); + +CREATE OR REPLACE FUNCTION flows.get_flow_checkpoints( + IN i_flow_id BIGINT, + IN i_checkpoints_limit INT DEFAULT 5, + IN i_offset BIGINT DEFAULT 0, + IN i_checkpoint_name TEXT DEFAULT NULL, + IN i_checkpoint_properties JSONB DEFAULT NULL, + IN i_latest_first BOOLEAN DEFAULT TRUE, + IN i_from_time TIMESTAMP WITH TIME ZONE DEFAULT NULL, + IN i_to_time TIMESTAMP WITH TIME ZONE DEFAULT NULL, + OUT status INTEGER, + OUT status_text TEXT, + OUT id_checkpoint UUID, + OUT checkpoint_name TEXT, + OUT checkpoint_author TEXT, + OUT measured_by_atum_agent BOOLEAN, + OUT measure_name TEXT, + OUT measured_columns TEXT[], + OUT measurement_value JSONB, + OUT checkpoint_start_time TIMESTAMP WITH TIME ZONE, + OUT checkpoint_end_time TIMESTAMP WITH TIME ZONE, + OUT id_partitioning BIGINT, + OUT partitioning JSONB, + OUT partitioning_author TEXT, + OUT has_more BOOLEAN +) RETURNS SETOF record AS +-------------------------------------------------------------------------------------------------------------------- +-- +-- Function: flows.get_flow_checkpoints(8) +-- Retrieves all checkpoints (measures and their measurement details) related to an input flow (and checkpoint +-- name, checkpoint properties and/or process start time window, if specified). +-- +-- Note: a single row returned from this function doesn't contain all data related to a single checkpoint - it only +-- represents one measure associated with a checkpoint. So even if only a single checkpoint would be retrieved, +-- this function can potentially return multiple rows. +-- +-- Parameters: +-- i_flow_id - ID of the flow for which checkpoints are to be retrieved +-- i_checkpoints_limit - (optional) maximum number of checkpoints to return, returns all of them if NULL +-- i_offset - (optional) offset of the first checkpoint to return +-- i_checkpoint_name - (optional) if specified, returns data related to particular checkpoint's name +-- i_checkpoint_properties - (optional) JSON object mapping a property name to an array of accepted values, +-- e.g. {"executionID": ["a", "b"]}; returns only checkpoints that, for every given +-- property name, have that property with one of the accepted values +-- i_latest_first - (optional) if true (default), checkpoints are ordered by process_start_time +-- in descending order (latest first); if false, in ascending order +-- i_from_time - (optional) if specified, returns only checkpoints with process_start_time >= i_from_time +-- i_to_time - (optional) if specified, returns only checkpoints with process_start_time < i_to_time +-- +-- Note: i_checkpoints_limit and i_offset are used for pagination purposes; +-- checkpoints are ordered by process_start_time (descending by default, see i_latest_first) +-- and then by id_checkpoint in ascending order as a tie-breaker +-- +-- Returns: +-- status - Status code +-- status_text - Status text +-- id_checkpoint - ID of retrieved checkpoint +-- checkpoint_name - Name of the retrieved checkpoint +-- checkpoint_author - Author of the checkpoint +-- measured_by_atum_agent - Flag indicating whether the checkpoint was measured by Atum Agent +-- (if false, data supplied manually) +-- measure_name - measure name associated with a given checkpoint +-- measured_columns - measure columns associated with a given checkpoint +-- measurement_value - measurement details associated with a given checkpoint +-- checkpoint_start_time - Time of the checkpoint +-- checkpoint_end_time - End time of the checkpoint computation +-- id_partitioning - ID of the partitioning +-- partitioning - Partitioning value +-- partitioning_author - Author of the partitioning +-- has_more - Flag indicating whether there are more checkpoints available, always `false` if `i_limit` is NULL +-- +-- Status codes: +-- 11 - OK +-- 42 - Flow not found +--------------------------------------------------------------------------------------------------- +$$ +DECLARE + _has_more BOOLEAN; + _latest_first BOOLEAN := coalesce(i_latest_first, TRUE); +BEGIN + -- Check if the flow exists by querying the partitioning_to_flow table. + -- Rationale: + -- This table is preferred over the flows table because: + -- 1. Every flow has at least one record in partitioning_to_flow. + -- 2. This table is used in subsequent queries, providing a caching advantage. + -- 3. Improves performance by reducing the need to query the flows table directly. + PERFORM 1 FROM flows.partitioning_to_flow WHERE fk_flow = i_flow_id; + IF NOT FOUND THEN + status := 42; + status_text := 'Flow not found'; + RETURN NEXT; + RETURN; + END IF; + + -- Determine if there are more checkpoints than the limit + IF i_checkpoints_limit IS NOT NULL THEN + SELECT count(*) > i_checkpoints_limit + FROM (SELECT 1 + FROM runs.checkpoints C + JOIN flows.partitioning_to_flow PF ON C.fk_partitioning = PF.fk_partitioning + WHERE PF.fk_flow = i_flow_id + AND (i_checkpoint_name IS NULL OR C.checkpoint_name = i_checkpoint_name) + AND (i_from_time IS NULL OR C.process_start_time >= i_from_time) + AND (i_to_time IS NULL OR C.process_start_time < i_to_time) + AND ( + i_checkpoint_properties IS NULL + OR NOT EXISTS ( + SELECT 1 + FROM jsonb_each(i_checkpoint_properties) AS flt(k, v) + WHERE NOT EXISTS ( + SELECT 1 + FROM runs.checkpoint_properties CP + WHERE CP.fk_checkpoint = C.id_checkpoint + AND CP.property_name = flt.k + AND (v ? CP.property_value) + ) + ) + ) + ORDER BY CASE + WHEN _latest_first THEN C.process_start_time + END DESC, + CASE + WHEN NOT _latest_first THEN C.process_start_time + END ASC, + C.id_checkpoint ASC + LIMIT i_checkpoints_limit + 1 OFFSET i_offset) s + INTO _has_more; + ELSE + _has_more := false; + END IF; + + -- Retrieve the checkpoints and their associated measurements + RETURN QUERY + WITH limited_checkpoints AS (SELECT C.id_checkpoint, + C.fk_partitioning, + C.checkpoint_name, + C.created_by, + C.measured_by_atum_agent, + C.process_start_time, + C.process_end_time + FROM runs.checkpoints C + JOIN flows.partitioning_to_flow PF ON C.fk_partitioning = PF.fk_partitioning + WHERE PF.fk_flow = i_flow_id + AND (i_checkpoint_name IS NULL OR C.checkpoint_name = i_checkpoint_name) + AND (i_from_time IS NULL OR C.process_start_time >= i_from_time) + AND (i_to_time IS NULL OR C.process_start_time < i_to_time) + AND ( + i_checkpoint_properties IS NULL + OR NOT EXISTS ( + SELECT 1 + FROM jsonb_each(i_checkpoint_properties) AS flt(k, v) + WHERE NOT EXISTS ( + SELECT 1 + FROM runs.checkpoint_properties CP + WHERE CP.fk_checkpoint = C.id_checkpoint + AND CP.property_name = flt.k + AND (v ? CP.property_value) + ) + ) + ) + ORDER BY CASE + WHEN _latest_first THEN C.process_start_time + END DESC, + CASE + WHEN NOT _latest_first THEN C.process_start_time + END ASC, + C.id_checkpoint ASC + LIMIT i_checkpoints_limit OFFSET i_offset) + SELECT 11 AS status, + 'OK' AS status_text, + LC.id_checkpoint, + LC.checkpoint_name, + LC.created_by AS checkpoint_author, + LC.measured_by_atum_agent, + MD.measure_name, + MD.measured_columns, + M.measurement_value, + LC.process_start_time AS checkpoint_start_time, + LC.process_end_time AS checkpoint_end_time, + LC.fk_partitioning AS id_partitioning, + P.partitioning AS partitioning, + P.created_by AS partitioning_author, + _has_more AS has_more + FROM limited_checkpoints LC + INNER JOIN + runs.measurements M ON LC.id_checkpoint = M.fk_checkpoint + INNER JOIN + runs.measure_definitions MD ON M.fk_measure_definition = MD.id_measure_definition + INNER JOIN + runs.partitionings P ON LC.fk_partitioning = P.id_partitioning + ORDER BY CASE + WHEN _latest_first THEN LC.process_start_time + END DESC, + CASE + WHEN NOT _latest_first THEN LC.process_start_time + END ASC, + LC.id_checkpoint ASC; +END; +$$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER; + +ALTER FUNCTION flows.get_flow_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE) OWNER TO atum_owner; +GRANT EXECUTE ON FUNCTION flows.get_flow_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE) TO atum_owner; diff --git a/database/src/main/postgres/flows/V0.8.0.5__get_flow_partitionings.sql b/database/src/main/postgres/flows/V0.8.0.5__get_flow_partitionings.sql new file mode 100644 index 000000000..fd9e5dd15 --- /dev/null +++ b/database/src/main/postgres/flows/V0.8.0.5__get_flow_partitionings.sql @@ -0,0 +1,98 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE OR REPLACE FUNCTION flows.get_flow_partitionings( + IN i_flow_id BIGINT, + IN i_limit INT DEFAULT 5, + IN i_offset BIGINT DEFAULT 0, + OUT status INTEGER, + OUT status_text TEXT, + OUT id BIGINT, + OUT partitioning JSONB, + OUT author TEXT, + OUT has_more BOOLEAN +) RETURNS SETOF record AS +------------------------------------------------------------------------------- +-- +-- Function: flows.get_flow_partitionings(3) +-- Retrieves all partitionings associated with the input flow. +-- +-- Note: partitionings are ordered by id_partitioning, then created_at descending. +-- +-- Parameters: +-- i_flow_id - flow id to use for identifying the partitionings that will be retrieved +-- i_limit - (optional) maximum number of partitionings to return, default is 5 +-- i_offset - (optional) offset to use for pagination, default is 0 +-- +-- Returns: +-- status - Status code +-- status_text - Status text +-- id - ID of retrieved partitioning +-- partitioning - Partitioning value +-- author - Author of the partitioning +-- has_more - Flag indicating if there are more partitionings available +-- +-- Status codes: +-- 11 - OK +-- 41 - Flow not found +-- +------------------------------------------------------------------------------- +$$ +DECLARE + _has_more BOOLEAN; +BEGIN + PERFORM 1 FROM flows.flows WHERE id_flow = i_flow_id; + IF NOT FOUND THEN + status := 41; + status_text := 'Flow not found'; + RETURN NEXT; + RETURN; + END IF; + + IF i_limit IS NOT NULL THEN + SELECT count(*) > i_limit + FROM (SELECT 1 + FROM flows.partitioning_to_flow PTF + WHERE PTF.fk_flow = i_flow_id + LIMIT i_limit + 1 OFFSET i_offset) s + INTO _has_more; + ELSE + _has_more := false; + END IF; + + + RETURN QUERY + SELECT + 11 AS status, + 'OK' AS status_text, + P.id_partitioning, + P.partitioning, + P.created_by, + _has_more + FROM + runs.partitionings P INNER JOIN + flows.partitioning_to_flow PF ON PF.fk_partitioning = P.id_partitioning + WHERE + PF.fk_flow = i_flow_id + ORDER BY + P.id_partitioning, + P.created_at DESC + LIMIT i_limit OFFSET i_offset; +END; +$$ +LANGUAGE plpgsql VOLATILE SECURITY DEFINER; + +GRANT EXECUTE ON FUNCTION flows.get_flow_partitionings(BIGINT, INT, BIGINT) TO atum_owner; diff --git a/database/src/main/postgres/runs/V0.8.0.2__get_partitioning_checkpoints.sql b/database/src/main/postgres/runs/V0.8.0.2__get_partitioning_checkpoints.sql new file mode 100644 index 000000000..3ccc4103c --- /dev/null +++ b/database/src/main/postgres/runs/V0.8.0.2__get_partitioning_checkpoints.sql @@ -0,0 +1,201 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- The previous overload is dropped so that the new signature (JSONB multi-value checkpoint properties filter and +-- the process start time window) is unambiguous when the function is called with named or defaulted arguments. +DROP FUNCTION IF EXISTS runs.get_partitioning_checkpoints(BIGINT, INT, BIGINT, TEXT, HSTORE, BOOLEAN); + +CREATE OR REPLACE FUNCTION runs.get_partitioning_checkpoints( + IN i_partitioning_id BIGINT, + IN i_checkpoints_limit INT DEFAULT 5, + IN i_offset BIGINT DEFAULT 0, + IN i_checkpoint_name TEXT DEFAULT NULL, + IN i_checkpoint_properties JSONB DEFAULT NULL, + IN i_latest_first BOOLEAN DEFAULT TRUE, + IN i_from_time TIMESTAMP WITH TIME ZONE DEFAULT NULL, + IN i_to_time TIMESTAMP WITH TIME ZONE DEFAULT NULL, + OUT status INTEGER, + OUT status_text TEXT, + OUT id_checkpoint UUID, + OUT checkpoint_name TEXT, + OUT checkpoint_author TEXT, + OUT measured_by_atum_agent BOOLEAN, + OUT measure_name TEXT, + OUT measured_columns TEXT[], + OUT measurement_value JSONB, + OUT checkpoint_start_time TIMESTAMP WITH TIME ZONE, + OUT checkpoint_end_time TIMESTAMP WITH TIME ZONE, + OUT has_more BOOLEAN +) RETURNS SETOF record AS +-------------------------------------------------------------------------------------------------------------------- +-- +-- Function: runs.get_partitioning_checkpoints(8) +-- Retrieves all checkpoints (measures and their measurement details) related to an input partitioning (and +-- checkpoint name, checkpoint properties and/or process start time window, if specified). +-- +-- Note: a single row returned from this function doesn't contain all data related to a single checkpoint - it only +-- represents one measure associated with a checkpoint. So even if only a single checkpoint would be retrieved, +-- this function can potentially return multiple rows. +-- +-- Parameters: +-- i_partitioning_id - ID of the partitioning for which checkpoints are to be retrieved +-- i_checkpoints_limit - (optional) maximum number of checkpoints to return, returns all of them if NULL +-- i_offset - (optional) offset of the first checkpoint to return +-- i_checkpoint_name - (optional) if specified, returns data related to particular checkpoint's name +-- i_checkpoint_properties - (optional) JSON object mapping a property name to an array of accepted values, +-- e.g. {"executionID": ["a", "b"]}; returns only checkpoints that, for every given +-- property name, have that property with one of the accepted values +-- i_latest_first - (optional) if true (default), checkpoints are ordered by process_start_time +-- in descending order (latest first); if false, in ascending order +-- i_from_time - (optional) if specified, returns only checkpoints with process_start_time >= i_from_time +-- i_to_time - (optional) if specified, returns only checkpoints with process_start_time < i_to_time +-- +-- Note: i_checkpoints_limit and i_offset are used for pagination purposes; +-- checkpoints are ordered by process_start_time (descending by default, see i_latest_first) +-- and then by id_checkpoint in ascending order as a tie-breaker +-- +-- Returns: +-- status - Status code +-- status_text - Status text +-- id_checkpoint - ID of retrieved checkpoint +-- checkpoint_name - Name of the retrieved checkpoint +-- checkpoint_author - Author of the checkpoint +-- measured_by_atum_agent - Flag indicating whether the checkpoint was measured by Atum Agent +-- (if false, data supplied manually) +-- measure_name - measure name associated with a given checkpoint +-- measured_columns - measure columns associated with a given checkpoint +-- measurement_value - measurement details associated with a given checkpoint +-- checkpoint_start_time - Time of the checkpoint +-- checkpoint_end_time - End time of the checkpoint computation +-- has_more - Flag indicating whether there are more checkpoints available +-- +-- Status codes: +-- 11 - OK +-- 41 - Partitioning not found +--------------------------------------------------------------------------------------------------- +$$ +DECLARE + _has_more BOOLEAN; + _latest_first BOOLEAN := coalesce(i_latest_first, TRUE); +BEGIN + PERFORM 1 FROM runs.partitionings WHERE id_partitioning = i_partitioning_id; + IF NOT FOUND THEN + status := 41; + status_text := 'Partitioning not found'; + RETURN NEXT; + RETURN; + END IF; + + -- Determine if there are more checkpoints than the limit + IF i_checkpoints_limit IS NOT NULL THEN + SELECT count(*) > i_checkpoints_limit + FROM (SELECT 1 + FROM runs.checkpoints C + WHERE C.fk_partitioning = i_partitioning_id + AND (i_checkpoint_name IS NULL OR C.checkpoint_name = i_checkpoint_name) + AND (i_from_time IS NULL OR C.process_start_time >= i_from_time) + AND (i_to_time IS NULL OR C.process_start_time < i_to_time) + AND ( + i_checkpoint_properties IS NULL + OR NOT EXISTS ( + SELECT 1 + FROM jsonb_each(i_checkpoint_properties) AS flt(k, v) + WHERE NOT EXISTS ( + SELECT 1 + FROM runs.checkpoint_properties CP + WHERE CP.fk_checkpoint = C.id_checkpoint + AND CP.property_name = flt.k + AND (v ? CP.property_value) + ) + ) + ) + ORDER BY CASE + WHEN _latest_first THEN C.process_start_time + END DESC, + CASE + WHEN NOT _latest_first THEN C.process_start_time + END ASC, + C.id_checkpoint ASC + LIMIT i_checkpoints_limit + 1 OFFSET i_offset) s + INTO _has_more; + ELSE + _has_more := false; + END IF; + + -- Retrieve the checkpoints and their associated measurements + RETURN QUERY + WITH limited_checkpoints AS (SELECT C.id_checkpoint, + C.checkpoint_name, + C.created_by, + C.measured_by_atum_agent, + C.process_start_time, + C.process_end_time + FROM runs.checkpoints C + WHERE C.fk_partitioning = i_partitioning_id + AND (i_checkpoint_name IS NULL OR C.checkpoint_name = i_checkpoint_name) + AND (i_from_time IS NULL OR C.process_start_time >= i_from_time) + AND (i_to_time IS NULL OR C.process_start_time < i_to_time) + AND ( + i_checkpoint_properties IS NULL + OR NOT EXISTS ( + SELECT 1 + FROM jsonb_each(i_checkpoint_properties) AS flt(k, v) + WHERE NOT EXISTS ( + SELECT 1 + FROM runs.checkpoint_properties CP + WHERE CP.fk_checkpoint = C.id_checkpoint + AND CP.property_name = flt.k + AND (v ? CP.property_value) + ) + ) + ) + ORDER BY CASE + WHEN _latest_first THEN C.process_start_time + END DESC, + CASE + WHEN NOT _latest_first THEN C.process_start_time + END ASC, + C.id_checkpoint ASC + LIMIT i_checkpoints_limit OFFSET i_offset) + SELECT 11 AS status, + 'OK' AS status_text, + LC.id_checkpoint, + LC.checkpoint_name, + LC.created_by AS checkpoint_author, + LC.measured_by_atum_agent, + MD.measure_name, + MD.measured_columns, + M.measurement_value, + LC.process_start_time AS checkpoint_start_time, + LC.process_end_time AS checkpoint_end_time, + _has_more AS has_more + FROM limited_checkpoints LC + INNER JOIN + runs.measurements M ON LC.id_checkpoint = M.fk_checkpoint + INNER JOIN + runs.measure_definitions MD ON M.fk_measure_definition = MD.id_measure_definition + ORDER BY CASE + WHEN _latest_first THEN LC.process_start_time + END DESC, + CASE + WHEN NOT _latest_first THEN LC.process_start_time + END ASC, + LC.id_checkpoint ASC; +END; +$$ LANGUAGE plpgsql VOLATILE SECURITY DEFINER; + +ALTER FUNCTION runs.get_partitioning_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE) OWNER TO atum_owner; +GRANT EXECUTE ON FUNCTION runs.get_partitioning_checkpoints(BIGINT, INT, BIGINT, TEXT, JSONB, BOOLEAN, TIMESTAMP WITH TIME ZONE, TIMESTAMP WITH TIME ZONE) TO atum_owner; diff --git a/database/src/main/postgres/runs/V0.8.0.3__checkpoint_properties_checkpoint_index.ddl b/database/src/main/postgres/runs/V0.8.0.3__checkpoint_properties_checkpoint_index.ddl new file mode 100644 index 000000000..bf7f54a0f --- /dev/null +++ b/database/src/main/postgres/runs/V0.8.0.3__checkpoint_properties_checkpoint_index.ddl @@ -0,0 +1,23 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Supports the checkpoint properties filter of runs.get_partitioning_checkpoints and flows.get_flow_checkpoints +-- (looked up per checkpoint and property name), and the retrieval of a checkpoint's properties. +-- +-- Built CONCURRENTLY so writes to runs.checkpoint_properties are not blocked. This file must contain only this +-- statement: Flyway then runs it outside of a transaction, which CONCURRENTLY requires. +-- If the build fails, it leaves an INVALID index behind; drop it +-- (DROP INDEX CONCURRENTLY runs.idx_checkpoint_properties_checkpoint_name) and re-run the migration. +CREATE INDEX CONCURRENTLY idx_checkpoint_properties_checkpoint_name ON runs.checkpoint_properties (fk_checkpoint, property_name); diff --git a/database/src/main/postgres/runs/V0.8.0.4__checkpoints_partitioning_start_time_index.ddl b/database/src/main/postgres/runs/V0.8.0.4__checkpoints_partitioning_start_time_index.ddl new file mode 100644 index 000000000..9e7d84004 --- /dev/null +++ b/database/src/main/postgres/runs/V0.8.0.4__checkpoints_partitioning_start_time_index.ddl @@ -0,0 +1,23 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Supports the process start time window of runs.get_partitioning_checkpoints and flows.get_flow_checkpoints: +-- checkpoints of each partitioning are range-scanned by process_start_time. +-- +-- Built CONCURRENTLY so writes to runs.checkpoints are not blocked. This file must contain only this statement: +-- Flyway then runs it outside of a transaction, which CONCURRENTLY requires. +-- If the build fails, it leaves an INVALID index behind; drop it (DROP INDEX CONCURRENTLY runs.checkpoints_idx2) +-- and re-run the migration. +CREATE INDEX CONCURRENTLY checkpoints_idx2 ON runs.checkpoints (fk_partitioning, process_start_time); diff --git a/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala index f5e21e89e..0ff920d9c 100644 --- a/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowCheckpointsIntegrationTests.scala @@ -3,7 +3,7 @@ package za.co.absa.atum.database.flows import io.circe.Json import io.circe.parser.parse import za.co.absa.balta.DBTestSuite -import za.co.absa.balta.classes.JsonBString +import za.co.absa.balta.classes.{JsonBString, QueryResult} import za.co.absa.balta.classes.setter.CustomDBType import java.time.OffsetDateTime @@ -521,185 +521,6 @@ class GetFlowCheckpointsIntegrationTests extends DBTestSuite { } } - test("getFlowCheckpointsV2 should return limited with checkpoints for a given flow") { - - val partitioningId: Long = Random.nextLong() - table("runs.partitionings").insert( - add("id_partitioning", partitioningId) - .add("partitioning", partitioning) - .add("created_by", "Joseph") - ) - - val flowId: Long = Random.nextLong() - table("flows.flows").insert( - add("id_flow", flowId) - .add("flow_name", "flowName") - .add("from_pattern", false) - .add("created_by", "Joseph") - .add("fk_primary_partitioning", partitioningId) - ) - - table("flows.partitioning_to_flow").insert( - add("fk_flow", flowId) - .add("fk_partitioning", partitioningId) - .add("created_by", "ObviouslySomeTest") - ) - - // Insert checkpoints and measure definitions - // checkpoint1 has a later start time so it sorts first with latest_first=TRUE - val checkpointId1 = UUID.randomUUID() - val startTime = OffsetDateTime.parse("1993-02-14T10:00:00Z") - val endTime = OffsetDateTime.parse("2024-04-24T10:00:00Z") - table("runs.checkpoints").insert( - add("id_checkpoint", checkpointId1) - .add("fk_partitioning", partitioningId) - .add("checkpoint_name", "CheckpointNameCntAndAvg") - .add("measured_by_atum_agent", true) - .add("process_start_time", startTime) - .add("process_end_time", endTime) - .add("created_by", "Joseph") - ) - - val checkpointId2 = UUID.randomUUID() - val startTimeOther = OffsetDateTime.parse("1992-02-14T10:00:00Z") - val endTimeOther = OffsetDateTime.parse("2023-04-24T10:00:00Z") - table("runs.checkpoints").insert( - add("id_checkpoint", checkpointId2) - .add("fk_partitioning", partitioningId) - .add("checkpoint_name", "CheckpointNameOther") - .add("measured_by_atum_agent", true) - .add("process_start_time", startTimeOther) - .add("process_end_time", endTimeOther) - .add("created_by", "Joseph") - ) - - // Insert measure definitions and measurements - val measureDefinitionAvgId: Long = Random.nextLong() - table("runs.measure_definitions").insert( - add("id_measure_definition", measureDefinitionAvgId) - .add("fk_partitioning", partitioningId) - .add("measure_name", "avg") - .add("measured_columns", CustomDBType("""{"a","b"}""", "TEXT[]")) - .add("created_by", "Joseph") - ) - - val measureDefinitionCntId: Long = Random.nextLong() - table("runs.measure_definitions").insert( - add("id_measure_definition", measureDefinitionCntId) - .add("fk_partitioning", partitioningId) - .add("measure_name", "cnt") - .add("measured_columns", CustomDBType("""{"col1"}""", "TEXT[]")) - .add("created_by", "Joseph") - ) - - val measureDefinitionOtherId: Long = Random.nextLong() - table("runs.measure_definitions").insert( - add("id_measure_definition", measureDefinitionOtherId) - .add("fk_partitioning", partitioningId) - .add("measure_name", "sum") - .add("measured_columns", CustomDBType("""{"colOther"}""", "TEXT[]")) - .add("created_by", "Joseph") - ) - - table("runs.measurements").insert( - add("fk_measure_definition", measureDefinitionCntId) - .add("fk_checkpoint", checkpointId1) - .add("measurement_value", measurementCnt) - ) - - table("runs.measurements").insert( - add("fk_measure_definition", measureDefinitionAvgId) - .add("fk_checkpoint", checkpointId1) - .add("measurement_value", measurementAvg) - ) - - table("runs.measurements").insert( - add("fk_measure_definition", measureDefinitionOtherId) - .add("fk_checkpoint", checkpointId2) - .add("measurement_value", measurementSum) - ) - - // Actual test execution and assertions - val actualMeasures: Seq[MeasuredDetails] = function(fncGetFlowCheckpointsV2) - .setParam("i_flow_id", flowId) - .setParam("i_checkpoints_limit", 2) - .setParam("i_offset", 0L) - .execute("checkpoint_name") { queryResult => - assert(queryResult.hasNext) - val row1 = queryResult.next() - assert(row1.getInt("status").contains(11)) - assert(row1.getString("status_text").contains("OK")) - assert(row1.getUUID("id_checkpoint").contains(checkpointId1)) - assert(row1.getString("checkpoint_name").contains("CheckpointNameCntAndAvg")) - assert(row1.getString("checkpoint_author").contains("Joseph")) - assert(row1.getBoolean("measured_by_atum_agent").contains(true)) - assert(row1.getOffsetDateTime("checkpoint_start_time").contains(startTime)) - assert(row1.getOffsetDateTime("checkpoint_end_time").contains(endTime)) - assert(queryResult.hasNext) - - val measure1 = MeasuredDetails( - row1.getString("measure_name").get, - row1.getArray[String]("measured_columns").map(_.toList).get, - row1.getJsonB("measurement_value").get - ) - - val row2 = queryResult.next() - assert(row2.getInt("status").contains(11)) - assert(row2.getString("status_text").contains("OK")) - assert(row2.getUUID("id_checkpoint").contains(checkpointId1)) - assert(row2.getString("checkpoint_name").contains("CheckpointNameCntAndAvg")) - assert(row2.getString("checkpoint_author").contains("Joseph")) - assert(row2.getBoolean("measured_by_atum_agent").contains(true)) - assert(row2.getOffsetDateTime("checkpoint_start_time").contains(startTime)) - assert(row2.getOffsetDateTime("checkpoint_end_time").contains(endTime)) - assert(queryResult.hasNext) - - val measure2 = MeasuredDetails( - row2.getString("measure_name").get, - row2.getArray[String]("measured_columns").map(_.toList).get, - row2.getJsonB("measurement_value").get - ) - - val row3 = queryResult.next() - assert(row3.getInt("status").contains(11)) - assert(row3.getString("status_text").contains("OK")) - assert(row3.getUUID("id_checkpoint").contains(checkpointId2)) - assert(row3.getString("checkpoint_name").contains("CheckpointNameOther")) - assert(row3.getString("checkpoint_author").contains("Joseph")) - assert(row3.getBoolean("measured_by_atum_agent").contains(true)) - assert(row3.getOffsetDateTime("checkpoint_start_time").contains(startTimeOther)) - assert(row3.getOffsetDateTime("checkpoint_end_time").contains(endTimeOther)) - - val measure3 = MeasuredDetails( - row3.getString("measure_name").get, - row3.getArray[String]("measured_columns").map(_.toList).get, - row3.getJsonB("measurement_value").get - ) - - assert(!queryResult.hasNext) - Seq(measure1, measure2, measure3) - } - - // Assertions for measures - assert(actualMeasures.map(_.measureName).toSet == Set("avg", "sum")) - assert(actualMeasures.map(_.measureColumns).toSet == Set(List("a", "b"), List("colOther"))) - - actualMeasures.foreach { currVal => - val currValStr = currVal.measurementValue.value - - currVal.measureName match { - case "avg" => - assert(currValStr.contains(""""value": "2.71"""")) - case "cnt" => - assert(currValStr.contains(""""value": "3"""")) - case "sum" => - assert(currValStr.contains(""""value": "3000"""")) - case other => - fail(s"Unexpected measure name: $other") - } - } - } - test("getFlowCheckpointsV2 should return no flows when flow_id is not found") { // Create a non-existent flowId that doesn't exist in the database @@ -800,15 +621,17 @@ class GetFlowCheckpointsIntegrationTests extends DBTestSuite { .add("property_value", "456") ) - def hstore(properties: Map[String, String]): CustomDBType = CustomDBType( - properties.map { case (k, v) => s""""$k"=>"$v"""" }.mkString(","), - "HSTORE" - ) + def jsonb(properties: Map[String, Seq[String]]): JsonBString = { + val inner = properties.map { case (k, vs) => + s"\"$k\": [${vs.map(v => s"\"$v\"").mkString(", ")}]" + }.mkString(", ") + JsonBString(s"{$inner}") + } // Filtering by jobId=123 returns only the matching checkpoint function(fncGetFlowCheckpointsV2) .setParam("i_flow_id", flowId) - .setParam("i_checkpoint_properties", hstore(Map("jobId" -> "123"))) + .setParam("i_checkpoint_properties", jsonb(Map("jobId" -> Seq("123")))) .execute { queryResult => assert(queryResult.hasNext) val row = queryResult.next() @@ -820,7 +643,7 @@ class GetFlowCheckpointsIntegrationTests extends DBTestSuite { // Requiring both jobId=123 and env=prod still returns the matching checkpoint (AND semantics) function(fncGetFlowCheckpointsV2) .setParam("i_flow_id", flowId) - .setParam("i_checkpoint_properties", hstore(Map("jobId" -> "123", "env" -> "prod"))) + .setParam("i_checkpoint_properties", jsonb(Map("jobId" -> Seq("123"), "env" -> Seq("prod")))) .execute { queryResult => assert(queryResult.hasNext) val row = queryResult.next() @@ -831,7 +654,7 @@ class GetFlowCheckpointsIntegrationTests extends DBTestSuite { // Requiring jobId=123 AND a property the checkpoint doesn't have returns nothing (AND semantics) function(fncGetFlowCheckpointsV2) .setParam("i_flow_id", flowId) - .setParam("i_checkpoint_properties", hstore(Map("jobId" -> "123", "env" -> "dev"))) + .setParam("i_checkpoint_properties", jsonb(Map("jobId" -> Seq("123"), "env" -> Seq("dev")))) .execute { queryResult => assert(!queryResult.hasNext) } @@ -839,12 +662,154 @@ class GetFlowCheckpointsIntegrationTests extends DBTestSuite { // Filtering by a value that no checkpoint has returns nothing function(fncGetFlowCheckpointsV2) .setParam("i_flow_id", flowId) - .setParam("i_checkpoint_properties", hstore(Map("jobId" -> "999"))) + .setParam("i_checkpoint_properties", jsonb(Map("jobId" -> Seq("999")))) .execute { queryResult => assert(!queryResult.hasNext) } } + test("getFlowCheckpointsV2 should return only checkpoints within the process start time window, across partitionings") { + + val otherPartitioning = JsonBString( + """ + |{ + | "version": 1, + | "keys": ["keyX", "keyY", "keyZ"], + | "keysToValuesMap": { + | "keyX": "value1", + | "keyZ": "value3", + | "keyY": "otherValue2" + | } + |} + |""".stripMargin + ) + + val partitioningIdA: Long = Random.nextLong() + val partitioningIdB: Long = Random.nextLong() + table("runs.partitionings").insert( + add("id_partitioning", partitioningIdA) + .add("partitioning", partitioning) + .add("created_by", "Joseph") + ) + table("runs.partitionings").insert( + add("id_partitioning", partitioningIdB) + .add("partitioning", otherPartitioning) + .add("created_by", "Joseph") + ) + + val flowId: Long = Random.nextLong() + table("flows.flows").insert( + add("id_flow", flowId) + .add("flow_name", "flowNameTimeWindow") + .add("from_pattern", false) + .add("created_by", "Joseph") + .add("fk_primary_partitioning", partitioningIdA) + ) + Seq(partitioningIdA, partitioningIdB).foreach { partitioningId => + table("flows.partitioning_to_flow").insert( + add("fk_flow", flowId) + .add("fk_partitioning", partitioningId) + .add("created_by", "ObviouslySomeTest") + ) + } + + val measureDefinitionIdA: Long = Random.nextLong() + val measureDefinitionIdB: Long = Random.nextLong() + Seq(partitioningIdA -> measureDefinitionIdA, partitioningIdB -> measureDefinitionIdB).foreach { + case (partitioningId, measureDefinitionId) => + table("runs.measure_definitions").insert( + add("id_measure_definition", measureDefinitionId) + .add("fk_partitioning", partitioningId) + .add("measure_name", "cnt") + .add("measured_columns", CustomDBType("""{"col1"}""", "TEXT[]")) + .add("created_by", "Joseph") + ) + } + + // one checkpoint at the start of each month, alternating between the two partitionings of the flow + case class TestCheckpoint(id: UUID, partitioningId: Long, measureDefinitionId: Long, start: OffsetDateTime, executionId: String) + val jan = TestCheckpoint(UUID.randomUUID(), partitioningIdA, measureDefinitionIdA, OffsetDateTime.parse("2026-01-01T00:00:00Z"), "e1") + val feb = TestCheckpoint(UUID.randomUUID(), partitioningIdB, measureDefinitionIdB, OffsetDateTime.parse("2026-02-01T00:00:00Z"), "e1") + val mar = TestCheckpoint(UUID.randomUUID(), partitioningIdA, measureDefinitionIdA, OffsetDateTime.parse("2026-03-01T00:00:00Z"), "e2") + val apr = TestCheckpoint(UUID.randomUUID(), partitioningIdB, measureDefinitionIdB, OffsetDateTime.parse("2026-04-01T00:00:00Z"), "e3") + Seq(jan, feb, mar, apr).foreach { checkpoint => + table("runs.checkpoints").insert( + add("id_checkpoint", checkpoint.id) + .add("fk_partitioning", checkpoint.partitioningId) + .add("checkpoint_name", "CheckpointName") + .add("measured_by_atum_agent", true) + .add("process_start_time", checkpoint.start) + .add("created_by", "Joseph") + ) + table("runs.measurements").insert( + add("fk_measure_definition", checkpoint.measureDefinitionId) + .add("fk_checkpoint", checkpoint.id) + .add("measurement_value", measurementCnt) + ) + table("runs.checkpoint_properties").insert( + add("fk_checkpoint", checkpoint.id) + .add("property_name", "executionID") + .add("property_value", checkpoint.executionId) + ) + } + + def checkpointsOf(queryResult: QueryResult): Seq[(UUID, Long)] = + queryResult.map(row => (row.getUUID("id_checkpoint").get, row.getLong("id_partitioning").get)).toList + + // [feb, apr) spans both partitionings of the flow, latest first + function(fncGetFlowCheckpointsV2) + .setParam("i_flow_id", flowId) + .setParam("i_from_time", feb.start) + .setParam("i_to_time", apr.start) + .execute { queryResult => + assert(checkpointsOf(queryResult) == Seq((mar.id, partitioningIdA), (feb.id, partitioningIdB))) + } + + // the same window, earliest first + function(fncGetFlowCheckpointsV2) + .setParam("i_flow_id", flowId) + .setParam("i_from_time", feb.start) + .setParam("i_to_time", apr.start) + .setParam("i_latest_first", false) + .execute { queryResult => + assert(checkpointsOf(queryResult).map(_._1) == Seq(feb.id, mar.id)) + } + + // combined with a multi-value checkpoint properties filter: executionID IN (e1, e3) since February + function(fncGetFlowCheckpointsV2) + .setParam("i_flow_id", flowId) + .setParam("i_checkpoint_properties", JsonBString("""{"executionID": ["e1", "e3"]}""")) + .setParam("i_from_time", feb.start) + .execute { queryResult => + assert(checkpointsOf(queryResult).map(_._1) == Seq(apr.id, feb.id)) + } + + // paging within the window: has_more is computed over the window only + function(fncGetFlowCheckpointsV2) + .setParam("i_flow_id", flowId) + .setParam("i_checkpoints_limit", 1) + .setParam("i_from_time", feb.start) + .setParam("i_to_time", apr.start) + .execute { queryResult => + val row = queryResult.next() + assert(row.getUUID("id_checkpoint").contains(mar.id)) + assert(row.getBoolean("has_more").contains(true)) + assert(!queryResult.hasNext) + } + function(fncGetFlowCheckpointsV2) + .setParam("i_flow_id", flowId) + .setParam("i_checkpoints_limit", 1) + .setParam("i_offset", 1L) + .setParam("i_from_time", feb.start) + .setParam("i_to_time", apr.start) + .execute { queryResult => + val row = queryResult.next() + assert(row.getUUID("id_checkpoint").contains(feb.id)) + assert(row.getBoolean("has_more").contains(false)) + assert(!queryResult.hasNext) + } + } + private def parseJsonBStringOrThrow(jsonBString: JsonBString): Json = { parse(jsonBString.value).getOrElse(throw new Exception("Failed to parse JsonBString to Json")) } diff --git a/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowPartitioningsIntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowPartitioningsIntegrationTests.scala index 0a9745ce8..c8053fc6f 100644 --- a/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowPartitioningsIntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/flows/GetFlowPartitioningsIntegrationTests.scala @@ -174,6 +174,72 @@ class GetFlowPartitioningsIntegrationTests extends DBTestSuite { } } + Seq( + ("first page", Some(2), 0L, Seq(0, 1), true), + ("middle page", Some(2), 2L, Seq(2, 3), true), + ("last partial page", Some(2), 4L, Seq(4), false), + ("exact final full page", Some(2), 3L, Seq(3, 4), false), + ("offset at the end", Some(2), 5L, Seq.empty[Int], false), + ("offset beyond the end", Some(2), 6L, Seq.empty[Int], false), + ("unlimited first page", None, 0L, Seq(0, 1, 2, 3, 4), false), + ("unlimited page with an offset", None, 2L, Seq(2, 3, 4), false) + ).foreach { case (description, limit, offset, expectedIndices, hasMore) => + test(s"Returns ordered partitionings and has_more for $description") { + val partitionings = (0 until 5).map { index => + val partitioning = JsonBString( + s"""{"version":1,"keys":["page"],"keysToValuesMap":{"page":"$index"}}""" + ) + table(partitioningsTable).insert(add("partitioning", partitioning).add("created_by", "Pagination author")) + val id: Long = table(partitioningsTable) + .fieldValue("partitioning", partitioning, "id_partitioning").get.get + (id, partitioning) + }.sortBy(_._1) + + val flowId = function(createFlowFn) + .setParam("i_fk_partitioning", partitionings.last._1) + .setParam("i_by_user", "Pagination author") + .execute { queryResult => + val row = queryResult.next() + assert(row.getInt("status").contains(11)) + row.getLong("id_flow").get + } + + partitionings.init.reverse.foreach { case (id, _) => + function(addToParentFlowsFn) + .setParam("i_fk_parent_partitioning", partitionings.last._1) + .setParam("i_fk_partitioning", id) + .setParam("i_by_user", "Pagination author") + .execute { queryResult => + assert(queryResult.next().getInt("status").contains(11)) + } + } + + val pageQuery = function(getFlowPartitioningsFn) + .setParam("i_flow_id", flowId) + .setParam("i_offset", offset) + val limitedQuery = limit match { + case Some(value) => pageQuery.setParam("i_limit", value) + case None => pageQuery.setParamNull("i_limit") + } + + limitedQuery.execute { queryResult => + expectedIndices.foreach { index => + assert(queryResult.hasNext) + val row = queryResult.next() + val (expectedId, expectedPartitioning) = partitionings(index) + assert(row.getInt("status").contains(11)) + assert(row.getString("status_text").contains("OK")) + assert(row.getLong("id").contains(expectedId)) + assert(parseJsonBStringOrThrow(row.getJsonB("partitioning").get) == + parseJsonBStringOrThrow(expectedPartitioning)) + assert(row.getString("author").contains("Pagination author")) + assert(row.getBoolean("has_more").contains(hasMore)) + } + assert(!queryResult.hasNext) + } + } + } + test("Fails for non-existent flow"){ function(getFlowPartitioningsFn) .setParam("i_flow_id", 999999) diff --git a/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala b/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala index 117d8b6d0..aefd225e5 100644 --- a/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala +++ b/database/src/test/scala/za/co/absa/atum/database/runs/GetPartitioningCheckpointsIntegrationTests.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.database.runs import za.co.absa.balta.DBTestSuite -import za.co.absa.balta.classes.JsonBString +import za.co.absa.balta.classes.{JsonBString, QueryResult} import za.co.absa.balta.classes.setter.CustomDBType import java.time.OffsetDateTime @@ -280,19 +280,6 @@ class GetPartitioningCheckpointsIntegrationTests extends DBTestSuite { assert(!queryResult.hasNext) } - function(fncGetPartitioningCheckpoints) - .setParam("i_partitioning_id", 0L) - .setParam("i_checkpoints_limit", 2) - .setParam("i_offset", 1) - .setParam("i_checkpoint_name", i_checkpoint_name) - .execute { queryResult => - assert(queryResult.hasNext) - val result = queryResult.next() - assert(result.getInt("status").contains(41)) - assert(result.getString("status_text").contains("Partitioning not found")) - assert(!queryResult.hasNext) - } - function(fncGetPartitioningCheckpoints) .setParam("i_partitioning_id", fkPartitioning1) .setParam("i_checkpoints_limit", 1) @@ -390,15 +377,17 @@ class GetPartitioningCheckpointsIntegrationTests extends DBTestSuite { .add("property_value", "456") ) - def hstore(properties: Map[String, String]): CustomDBType = CustomDBType( - properties.map { case (k, v) => s""""$k"=>"$v"""" }.mkString(","), - "HSTORE" - ) + def jsonb(properties: Map[String, Seq[String]]): JsonBString = { + val inner = properties.map { case (k, vs) => + s"\"$k\": [${vs.map(v => s"\"$v\"").mkString(", ")}]" + }.mkString(", ") + JsonBString(s"{$inner}") + } // Filtering by jobId=123 returns only the matching checkpoint function(fncGetPartitioningCheckpoints) .setParam("i_partitioning_id", fkPartitioning) - .setParam("i_checkpoint_properties", hstore(Map("jobId" -> "123"))) + .setParam("i_checkpoint_properties", jsonb(Map("jobId" -> Seq("123")))) .execute { queryResult => assert(queryResult.hasNext) val row = queryResult.next() @@ -410,7 +399,7 @@ class GetPartitioningCheckpointsIntegrationTests extends DBTestSuite { // Filtering by a value that no checkpoint has returns nothing function(fncGetPartitioningCheckpoints) .setParam("i_partitioning_id", fkPartitioning) - .setParam("i_checkpoint_properties", hstore(Map("jobId" -> "999"))) + .setParam("i_checkpoint_properties", jsonb(Map("jobId" -> Seq("999")))) .execute { queryResult => assert(!queryResult.hasNext) } @@ -490,4 +479,118 @@ class GetPartitioningCheckpointsIntegrationTests extends DBTestSuite { } } + test("Returns only checkpoints within the process start time window") { + table("runs.partitionings").insert( + add("partitioning", partitioning1) + .add("created_by", "Daniel") + ) + val fkPartitioning: Long = table("runs.partitionings") + .fieldValue("partitioning", partitioning1, "id_partitioning").get.get + + table("runs.measure_definitions").insert( + add("id_measure_definition", id_measure_definition1) + .add("fk_partitioning", fkPartitioning) + .add("created_by", "Daniel") + .add("measure_name", "measure_1") + .add("measured_columns", measured_columns1) + ) + + // one checkpoint at the start of each month, January to April; named "odd" / "even" by month number + val times = Seq( + OffsetDateTime.parse("2026-01-01T00:00:00Z"), + OffsetDateTime.parse("2026-02-01T00:00:00Z"), + OffsetDateTime.parse("2026-03-01T00:00:00Z"), + OffsetDateTime.parse("2026-04-01T00:00:00Z") + ) + val checkpointIds = times.map(_ => UUID.randomUUID()) + times.zip(checkpointIds).zipWithIndex.foreach { case ((time, checkpointId), i) => + table("runs.checkpoints").insert( + add("id_checkpoint", checkpointId) + .add("fk_partitioning", fkPartitioning) + .add("checkpoint_name", if (i % 2 == 0) "odd" else "even") + .add("process_start_time", time) + .add("process_end_time", endTime) + .add("measured_by_atum_agent", true) + .add("created_by", "Daniel") + ) + table("runs.measurements").insert( + add("fk_checkpoint", checkpointId) + .add("fk_measure_definition", id_measure_definition1) + .add("measurement_value", measurement1) + ) + } + val Seq(jan, feb, mar, apr) = checkpointIds + + def checkpointsOf(queryResult: QueryResult): Seq[UUID] = + queryResult.map(_.getUUID("id_checkpoint").get).toList + + // i_from_time is inclusive + function(fncGetPartitioningCheckpoints) + .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_from_time", times(1)) + .execute { queryResult => + assert(checkpointsOf(queryResult) == Seq(apr, mar, feb)) + } + + // i_to_time is exclusive + function(fncGetPartitioningCheckpoints) + .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_to_time", times(2)) + .execute { queryResult => + assert(checkpointsOf(queryResult) == Seq(feb, jan)) + } + + // both bounds, earliest first + function(fncGetPartitioningCheckpoints) + .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_from_time", times(1)) + .setParam("i_to_time", times(3)) + .setParam("i_latest_first", false) + .execute { queryResult => + assert(checkpointsOf(queryResult) == Seq(feb, mar)) + } + + // paging within the window: has_more is computed over the window only + function(fncGetPartitioningCheckpoints) + .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_checkpoints_limit", 1) + .setParam("i_from_time", times(1)) + .setParam("i_to_time", times(3)) + .execute { queryResult => + val row = queryResult.next() + assert(row.getUUID("id_checkpoint").contains(mar)) + assert(row.getBoolean("has_more").contains(true)) + assert(!queryResult.hasNext) + } + function(fncGetPartitioningCheckpoints) + .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_checkpoints_limit", 1) + .setParam("i_offset", 1L) + .setParam("i_from_time", times(1)) + .setParam("i_to_time", times(3)) + .execute { queryResult => + val row = queryResult.next() + assert(row.getUUID("id_checkpoint").contains(feb)) + assert(row.getBoolean("has_more").contains(false)) + assert(!queryResult.hasNext) + } + + // combined with the checkpoint name filter + function(fncGetPartitioningCheckpoints) + .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_checkpoint_name", "odd") + .setParam("i_from_time", times(1)) + .execute { queryResult => + assert(checkpointsOf(queryResult) == Seq(mar)) + } + + // a window without checkpoints returns no rows + function(fncGetPartitioningCheckpoints) + .setParam("i_partitioning_id", fkPartitioning) + .setParam("i_from_time", OffsetDateTime.parse("2027-01-01T00:00:00Z")) + .execute { queryResult => + assert(!queryResult.hasNext) + } + } + } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala index 05e630a77..57f104d95 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala @@ -20,13 +20,12 @@ import sttp.client3.SttpBackend import sttp.monad.MonadError import sttp.monad.syntax._ import za.co.absa.atum.model.dto.{CheckpointWithPartitioningDTO, FlowDTO} -import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ import za.co.absa.atum.model.envelopes.SuccessResponse.{PaginatedResponse, SingleSuccessResponse} import za.co.absa.atum.model.types.basic.AtumPartitions import za.co.absa.atum.reader.core.RequestResult.RequestResult import za.co.absa.atum.model.ApiPaths._ import za.co.absa.atum.reader.core.{PartitioningIdProvider, Reader} -import za.co.absa.atum.reader.requests.QueryParamNames +import za.co.absa.atum.reader.requests.{CheckpointFilter, QueryParamNames} import za.co.absa.atum.reader.server.ServerConfig /** @@ -45,32 +44,33 @@ case class FlowReader[F[_]](mainFlowPartitioning: AtumPartitions)(implicit with PartitioningIdProvider[F] { /** - * Function to retrieve a page of checkpoints belonging to the flow. - * The checkpoints are ordered by their creation order. + * Function to retrieve a page of checkpoints belonging to the flow, optionally filtered. + * The checkpoints are ordered by their process start time, latest first unless the filter says otherwise. * - * @param pageSize - the size of the page (record count) to be returned - * @param offset - offset of the page (starting position) + * @param pageSize - the size of the page (record count) to be returned + * @param offset - offset of the page (starting position) * @param includeProperties - whether to include checkpoint properties in the response - * @return - a page of checkpoints + * @param filter - the checkpoints to return (name, properties, process start time window) and their order + * @return - a page of checkpoints */ def getCheckpointsPage( pageSize: Int = 10, offset: Long = 0, - includeProperties: Boolean = false + includeProperties: Boolean = false, + filter: CheckpointFilter = CheckpointFilter.empty ): F[RequestResult[PaginatedResponse[CheckpointWithPartitioningDTO]]] = { for { - mainPartitioningIdOrError <- partitioningId(mainFlowPartitioning) - flowIdOrError <- mapRequestResultF(mainPartitioningIdOrError, queryFlowId) + flowIdOrError <- flowId checkpointsOrError <- mapRequestResultF( flowIdOrError, - queryCheckpoints(_, None, Map.empty, pageSize, offset, includeProperties) + queryCheckpoints(_, filter, pageSize, offset, includeProperties) ) } yield checkpointsOrError } /** * Function to retrieve a page of checkpoints of the given name belonging to the flow. - * The checkpoints are ordered by their creation order. + * The checkpoints are ordered by their process start time, latest first. * * @param checkpointName - the name to filter with * @param pageSize - the size of the page (record count) to be returned @@ -78,26 +78,20 @@ case class FlowReader[F[_]](mainFlowPartitioning: AtumPartitions)(implicit * @param includeProperties - whether to include checkpoint properties in the response * @return - a page of checkpoints */ + @deprecated("Use getCheckpointsPage with filter = CheckpointFilter(name = Some(checkpointName))", "0.9.0") def getCheckpointsOfNamePage( checkpointName: String, pageSize: Int = 10, offset: Long = 0, includeProperties: Boolean = false ): F[RequestResult[PaginatedResponse[CheckpointWithPartitioningDTO]]] = { - for { - mainPartitioningIdOrError <- partitioningId(mainFlowPartitioning) - flowIdOrError <- mapRequestResultF(mainPartitioningIdOrError, queryFlowId) - checkpointsOrError <- mapRequestResultF( - flowIdOrError, - queryCheckpoints(_, Some(checkpointName), Map.empty, pageSize, offset, includeProperties) - ) - } yield checkpointsOrError + getCheckpointsPage(pageSize, offset, includeProperties, CheckpointFilter(name = Some(checkpointName))) } /** * Function to retrieve a page of checkpoints belonging to the flow that have all the given * checkpoint properties (matching both property name and value). - * The checkpoints are ordered by their creation order. + * The checkpoints are ordered by their process start time, latest first. * * @param checkpointProperties - the checkpoint properties (key-value pairs) to filter with; * a checkpoint is returned only if it has all of them @@ -106,20 +100,15 @@ case class FlowReader[F[_]](mainFlowPartitioning: AtumPartitions)(implicit * @param includeProperties - whether to include checkpoint properties in the response * @return - a page of checkpoints */ + @deprecated("Use getCheckpointsPage with filter = CheckpointFilter(properties = ...)", "0.9.0") def getCheckpointsByPropertiesPage( checkpointProperties: Map[String, String], pageSize: Int = 10, offset: Long = 0, includeProperties: Boolean = false ): F[RequestResult[PaginatedResponse[CheckpointWithPartitioningDTO]]] = { - for { - mainPartitioningIdOrError <- partitioningId(mainFlowPartitioning) - flowIdOrError <- mapRequestResultF(mainPartitioningIdOrError, queryFlowId) - checkpointsOrError <- mapRequestResultF( - flowIdOrError, - queryCheckpoints(_, None, checkpointProperties, pageSize, offset, includeProperties) - ) - } yield checkpointsOrError + val properties = checkpointProperties.map { case (propertyName, value) => propertyName -> Set(value) } + getCheckpointsPage(pageSize, offset, includeProperties, CheckpointFilter(properties = properties)) } /** @@ -137,25 +126,26 @@ case class FlowReader[F[_]](mainFlowPartitioning: AtumPartitions)(implicit } } + private def flowId: F[RequestResult[Long]] = { + for { + mainPartitioningIdOrError <- partitioningId(mainFlowPartitioning) + flowIdOrError <- mapRequestResultF(mainPartitioningIdOrError, queryFlowId) + } yield flowIdOrError + } + private def queryCheckpoints( flowId: Long, - checkpointName: Option[String], - checkpointProperties: Map[String, String], + filter: CheckpointFilter, limit: Int, offset: Long, includeProperties: Boolean ): F[RequestResult[PaginatedResponse[CheckpointWithPartitioningDTO]]] = { val endpoint = s"/$Api/$V2/${V2Paths.Flows}/$flowId/${V2Paths.Checkpoints}" - val propertiesParam: Option[(String, String)] = - if (checkpointProperties.isEmpty) None - else Some(QueryParamNames.CheckpointProperties -> checkpointProperties.asBase64EncodedJsonString) val params = Map( QueryParamNames.Limit -> limit.toString, QueryParamNames.Offset -> offset.toString, QueryParamNames.IncludeProperties -> includeProperties.toString - ) ++ - checkpointName.map(QueryParamNames.CheckpointName -> _) ++ - propertiesParam + ) ++ filter.toQueryParams getQuery(endpoint, params) } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala index a406f0035..5a38732f1 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala @@ -20,13 +20,12 @@ import sttp.client3.SttpBackend import sttp.monad.MonadError import sttp.monad.syntax._ import za.co.absa.atum.model.ApiPaths.{Api, V2, V2Paths} -import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ import za.co.absa.atum.model.dto.{AdditionalDataDTO, AdditionalDataItemDTO, AdditionalDataItemV2DTO, CheckpointV2DTO} import za.co.absa.atum.model.envelopes.SuccessResponse.{MultiSuccessResponse, PaginatedResponse, SingleSuccessResponse} import za.co.absa.atum.model.types.basic.AtumPartitions import za.co.absa.atum.reader.core.RequestResult.RequestResult import za.co.absa.atum.reader.core.{PartitioningIdProvider, Reader} -import za.co.absa.atum.reader.requests.QueryParamNames +import za.co.absa.atum.reader.requests.{CheckpointFilter, QueryParamNames} import za.co.absa.atum.reader.server.ServerConfig /** @@ -43,24 +42,26 @@ case class PartitioningReader[F[_]](partitioning: AtumPartitions)(implicit with PartitioningIdProvider[F] { /** - * Function to retrieve a page of checkpoints belonging to the partitioning. - * The checkpoints are ordered by their creation order. + * Function to retrieve a page of checkpoints belonging to the partitioning, optionally filtered. + * The checkpoints are ordered by their process start time, latest first unless the filter says otherwise. * - * @param pageSize - the size of the page (record count) to be returned - * @param offset - offset of the page (starting position) + * @param pageSize - the size of the page (record count) to be returned + * @param offset - offset of the page (starting position) * @param includeProperties - whether to include checkpoint properties in the response - * @return - a page of checkpoints + * @param filter - the checkpoints to return (name, properties, process start time window) and their order + * @return - a page of checkpoints */ def getCheckpointsPage( pageSize: Int = 10, offset: Long = 0, - includeProperties: Boolean = false + includeProperties: Boolean = false, + filter: CheckpointFilter = CheckpointFilter.empty ): F[RequestResult[PaginatedResponse[CheckpointV2DTO]]] = { for { partitioningIdOrError <- partitioningId(partitioning) checkpointsOrError <- mapRequestResultF( partitioningIdOrError, - queryCheckpoints(_, None, Map.empty, pageSize, offset, includeProperties) + queryCheckpoints(_, filter, pageSize, offset, includeProperties) ) } yield checkpointsOrError } @@ -69,7 +70,7 @@ case class PartitioningReader[F[_]](partitioning: AtumPartitions)(implicit * Function to retrieve a page of checkpoints of the given name belonging to the partitioning. (While the usual logic * would suggest, there would be only one checkpoint of a name, nothing prevents to have checkpoints of the same name; * also during reprocessing the checkpoints might multiply.) - * The checkpoints are ordered by their creation order. + * The checkpoints are ordered by their process start time, latest first. * * @param checkpointName - the name to filter with * @param pageSize - the size of the page (record count) to be returned @@ -77,25 +78,20 @@ case class PartitioningReader[F[_]](partitioning: AtumPartitions)(implicit * @param includeProperties - whether to include checkpoint properties in the response * @return - a page of checkpoints */ + @deprecated("Use getCheckpointsPage with filter = CheckpointFilter(name = Some(checkpointName))", "0.9.0") def getCheckpointsOfNamePage( checkpointName: String, pageSize: Int = 10, offset: Long = 0, includeProperties: Boolean = false ): F[RequestResult[PaginatedResponse[CheckpointV2DTO]]] = { - for { - partitioningIdOrError <- partitioningId(partitioning) - checkpointsOrError <- mapRequestResultF( - partitioningIdOrError, - queryCheckpoints(_, Some(checkpointName), Map.empty, pageSize, offset, includeProperties) - ) - } yield checkpointsOrError + getCheckpointsPage(pageSize, offset, includeProperties, CheckpointFilter(name = Some(checkpointName))) } /** * Function to retrieve a page of checkpoints belonging to the partitioning that have all the given * checkpoint properties (matching both property name and value). - * The checkpoints are ordered by their creation order. + * The checkpoints are ordered by their process start time, latest first. * * @param checkpointProperties - the checkpoint properties (key-value pairs) to filter with; * a checkpoint is returned only if it has all of them @@ -104,19 +100,15 @@ case class PartitioningReader[F[_]](partitioning: AtumPartitions)(implicit * @param includeProperties - whether to include checkpoint properties in the response * @return - a page of checkpoints */ + @deprecated("Use getCheckpointsPage with filter = CheckpointFilter(properties = ...)", "0.9.0") def getCheckpointsByPropertiesPage( checkpointProperties: Map[String, String], pageSize: Int = 10, offset: Long = 0, includeProperties: Boolean = false ): F[RequestResult[PaginatedResponse[CheckpointV2DTO]]] = { - for { - partitioningIdOrError <- partitioningId(partitioning) - checkpointsOrError <- mapRequestResultF( - partitioningIdOrError, - queryCheckpoints(_, None, checkpointProperties, pageSize, offset, includeProperties) - ) - } yield checkpointsOrError + val properties = checkpointProperties.map { case (propertyName, value) => propertyName -> Set(value) } + getCheckpointsPage(pageSize, offset, includeProperties, CheckpointFilter(properties = properties)) } /** @@ -150,23 +142,17 @@ case class PartitioningReader[F[_]](partitioning: AtumPartitions)(implicit private def queryCheckpoints( partitioningId: Long, - checkpointName: Option[String], - checkpointProperties: Map[String, String], + filter: CheckpointFilter, limit: Int, offset: Long, includeProperties: Boolean ): F[RequestResult[PaginatedResponse[CheckpointV2DTO]]] = { val endpoint = s"/$Api/$V2/${V2Paths.Partitionings}/$partitioningId/${V2Paths.Checkpoints}" - val propertiesParam: Option[(String, String)] = - if (checkpointProperties.isEmpty) None - else Some(QueryParamNames.CheckpointProperties -> checkpointProperties.asBase64EncodedJsonString) val params = Map( QueryParamNames.Limit -> limit.toString, QueryParamNames.Offset -> offset.toString, QueryParamNames.IncludeProperties -> includeProperties.toString - ) ++ - checkpointName.map(QueryParamNames.CheckpointName -> _) ++ - propertiesParam + ) ++ filter.toQueryParams getQuery(endpoint, params) } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala b/reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala new file mode 100644 index 000000000..4f2c13e23 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/requests/CheckpointFilter.scala @@ -0,0 +1,84 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.requests + +import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ + +import java.time.ZonedDateTime + +/** + * Filter of a checkpoints query. All its parts are optional and a checkpoint has to satisfy all the given ones. + * + * Example - checkpoints of two executions processed in the last two months, latest first: + * {{{ + * CheckpointFilter( + * properties = Map("executionID" -> Set("a", "b")), + * from = Some(ZonedDateTime.now().minusMonths(2)) + * ) + * }}} + * + * Note: the time window and properties with more than one value require an Atum server supporting them; older servers + * ignore the time window and reject multi-value properties. + * + * @param name - only checkpoints of this name + * @param properties - only checkpoints that, for every property name given, have that property with one of the given + * values, e.g. `Map("executionID" -> Set("a", "b"))` means `executionID IN (a, b)`; every property + * needs at least one value, otherwise an `IllegalArgumentException` is thrown + * @param from - only checkpoints with process start time at or after this time (inclusive) + * @param to - only checkpoints with process start time before this time (exclusive) + * @param latestFirst - order of the checkpoints by their process start time: latest first if `true`, earliest first if + * `false`, server default (latest first) if not specified + */ +case class CheckpointFilter( + name: Option[String] = None, + properties: Map[String, Set[String]] = Map.empty, + from: Option[ZonedDateTime] = None, + to: Option[ZonedDateTime] = None, + latestFirst: Option[Boolean] = None +) { + + require( + properties.values.forall(_.nonEmpty), + s"Checkpoint properties without any accepted value: ${properties.filter(_._2.isEmpty).keys.mkString(", ")}" + ) + + private[reader] def toQueryParams: Map[String, String] = { + name.map(QueryParamNames.CheckpointName -> _).toMap ++ + propertiesParam ++ + from.map(QueryParamNames.From -> _.toInstant.toString) ++ + to.map(QueryParamNames.To -> _.toInstant.toString) ++ + latestFirst.map(QueryParamNames.LatestFirst -> _.toString) + } + + private def propertiesParam: Option[(String, String)] = { + if (properties.isEmpty) { + None + } else if (properties.values.forall(_.size == 1)) { + // single values are sent in the original single-value format, understood by older servers too + val singleValues = properties.map { case (propertyName, values) => propertyName -> values.head } + Some(QueryParamNames.CheckpointProperties -> singleValues.asBase64EncodedJsonString) + } else { + val multiValues = properties.map { case (propertyName, values) => propertyName -> values.toSeq.sorted } + Some(QueryParamNames.CheckpointProperties -> multiValues.asBase64EncodedJsonString) + } + } + +} + +object CheckpointFilter { + val empty: CheckpointFilter = CheckpointFilter() +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala b/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala index f1bf111f9..e17baddcd 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala @@ -22,4 +22,7 @@ object QueryParamNames { final val CheckpointName = "checkpoint-name" final val CheckpointProperties = "checkpoint-properties" final val IncludeProperties = "include-properties" + final val LatestFirst = "latest-first" + final val From = "from" + final val To = "to" } diff --git a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala index b30611050..e881c25c4 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala @@ -31,10 +31,12 @@ import za.co.absa.atum.model.envelopes.SuccessResponse.PaginatedResponse import za.co.absa.atum.model.types.basic.{AtumPartitions, AtumPartitionsOps} import za.co.absa.atum.reader.FlowReaderUnitTests._ import za.co.absa.atum.reader.PartitioningReaderUnitTests.checkpointsResponseWithProperties +import za.co.absa.atum.reader.requests.CheckpointFilter import za.co.absa.atum.reader.server.ServerConfig import java.time.ZonedDateTime import java.util.UUID +import scala.annotation.nowarn class FlowReaderUnitTests extends AnyFunSuiteLike { private implicit val serverConfig: ServerConfig = ServerConfig.fromConfig() @@ -192,7 +194,7 @@ class FlowReaderUnitTests extends AnyFunSuiteLike { ) val reader = FlowReader(atumPartitions) - val result = reader.getCheckpointsOfNamePage("Test checkpoints 1") + val result = (reader.getCheckpointsOfNamePage("Test checkpoints 1"): @nowarn("cat=deprecation")) assert(result == Right(expectedData)) } @@ -273,7 +275,7 @@ class FlowReaderUnitTests extends AnyFunSuiteLike { ) val reader = FlowReader(atumPartitions) - val result = reader.getCheckpointsOfNamePage("Test checkpoints 1", includeProperties = true) + val result = (reader.getCheckpointsOfNamePage("Test checkpoints 1", includeProperties = true): @nowarn("cat=deprecation")) assert(result == Right(expectedData)) } @@ -299,10 +301,43 @@ class FlowReaderUnitTests extends AnyFunSuiteLike { val reader = FlowReader(atumPartitions) // base64url-encoded {"executionID":"019f8981-7868-79fc-81d3-8143a4706f8a"} - val result = reader.getCheckpointsByPropertiesPage(Map("executionID" -> "019f8981-7868-79fc-81d3-8143a4706f8a")) + val result = ( + reader.getCheckpointsByPropertiesPage(Map("executionID" -> "019f8981-7868-79fc-81d3-8143a4706f8a")): @nowarn("cat=deprecation") + ) assert(result.isRight) } + test("The flow checkpoints are queried with name, multi-value properties and time window in a single request") { + implicit val server: SttpBackendStub[Identity, capabilities.WebSockets] = SttpBackendStub.synchronous + .whenRequestMatchesPartial { + case r if r.uri.path.endsWith(List("partitionings")) => + Response.ok(partitioningResponse) + case r if r.uri.path.endsWith(List("partitionings", "7", "main-flow")) => + Response.ok(flowResponse) + case r if r.uri.path.endsWith(List("flows", "42", "checkpoints")) => + assert(r.uri.querySegments.contains(KeyValue("limit", "5"))) + assert(r.uri.querySegments.contains(KeyValue("offset", "10"))) + assert(r.uri.querySegments.contains(KeyValue("include-properties", "true"))) + assert(r.uri.querySegments.contains(KeyValue("checkpoint-name", "Test checkpoints 1"))) + assert(r.uri.querySegments.contains(KeyValue("checkpoint-properties", multiValuePropertiesEncoded))) + assert(r.uri.querySegments.contains(KeyValue("from", "2026-06-01T00:00:00Z"))) + assert(r.uri.querySegments.contains(KeyValue("to", "2026-08-01T00:00:00Z"))) + assert(r.uri.querySegments.contains(KeyValue("latest-first", "false"))) + Response.ok(checkpointsResponse) + } + + val filter = CheckpointFilter( + name = Some("Test checkpoints 1"), + properties = Map("executionID" -> Set("id2", "id1")), + from = Some(ZonedDateTime.parse("2026-06-01T02:00:00+02:00")), + to = Some(ZonedDateTime.parse("2026-08-01T00:00:00Z")), + latestFirst = Some(false) + ) + val result = FlowReader(AtumPartitions(List("a" -> "b", "c" -> "d"))) + .getCheckpointsPage(pageSize = 5, offset = 10, includeProperties = true, filter = filter) + assert(result.map(_.data.size) == Right(2)) + } + test("Instantiate FlowReader with implicit arguments passed explicitly") { val atumPartitions: AtumPartitions = AtumPartitions(List( "a" -> "b", @@ -336,6 +371,9 @@ class FlowReaderUnitTests extends AnyFunSuiteLike { object FlowReaderUnitTests { + // base64url-encoded {"executionID":["id1","id2"]} + private val multiValuePropertiesEncoded = "eyJleGVjdXRpb25JRCI6WyJpZDEiLCJpZDIiXX0=" + private val partitioningEncoded = "W3sia2V5IjoiYSIsInZhbHVlIjoiYiJ9LHsia2V5IjoiYyIsInZhbHVlIjoiZCJ9XQ==" private val partitioningResponse = diff --git a/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala index 37745ed07..9b7733d7f 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala @@ -39,10 +39,12 @@ import za.co.absa.atum.model.envelopes.Pagination import za.co.absa.atum.model.envelopes.SuccessResponse.PaginatedResponse import za.co.absa.atum.model.types.basic.AtumPartitions import za.co.absa.atum.reader.PartitioningReaderUnitTests._ +import za.co.absa.atum.reader.requests.CheckpointFilter import za.co.absa.atum.reader.server.ServerConfig import java.time.ZonedDateTime import java.util.UUID +import scala.annotation.nowarn class PartitioningReaderUnitTests extends AnyFunSuiteLike { private implicit val serverConfig: ServerConfig = ServerConfig.fromConfig() @@ -243,7 +245,7 @@ class PartitioningReaderUnitTests extends AnyFunSuiteLike { ) val reader = PartitioningReader(atumPartitions) - val result = reader.getCheckpointsOfNamePage("Test checkpoints 1") + val result = (reader.getCheckpointsOfNamePage("Test checkpoints 1"): @nowarn("cat=deprecation")) assert(result == Right(expected)) } @@ -269,10 +271,38 @@ class PartitioningReaderUnitTests extends AnyFunSuiteLike { val reader = PartitioningReader(atumPartitions) // base64url-encoded {"executionID":"019f8981-7868-79fc-81d3-8143a4706f8a"} - val result = reader.getCheckpointsByPropertiesPage(Map("executionID" -> "019f8981-7868-79fc-81d3-8143a4706f8a")) + val result = ( + reader.getCheckpointsByPropertiesPage(Map("executionID" -> "019f8981-7868-79fc-81d3-8143a4706f8a")): @nowarn("cat=deprecation") + ) assert(result.isRight) } + test("The partitioning checkpoints are queried with name, multi-value properties and time window in a single request") { + implicit val server: SttpBackendStub[Identity, capabilities.WebSockets] = SttpBackendStub.synchronous + .whenRequestMatchesPartial { + case r if r.uri.path.endsWith(List(V2Paths.Partitionings)) => + Response.ok(partitioningResponse) + case r if r.uri.path.endsWith(List(V2Paths.Partitionings, "7", V2Paths.Checkpoints)) => + assert(r.uri.querySegments.contains(KeyValue("limit", "10"))) + assert(r.uri.querySegments.contains(KeyValue("offset", "0"))) + assert(r.uri.querySegments.contains(KeyValue("checkpoint-name", "Test checkpoints 1"))) + // base64url-encoded {"executionID":["id1","id2"]} + assert(r.uri.querySegments.contains(KeyValue("checkpoint-properties", "eyJleGVjdXRpb25JRCI6WyJpZDEiLCJpZDIiXX0="))) + assert(r.uri.querySegments.contains(KeyValue("from", "2026-06-01T00:00:00Z"))) + assert(!r.uri.paramsMap.contains("to")) + assert(!r.uri.paramsMap.contains("latest-first")) + Response.ok(checkpointsResponse) + } + + val filter = CheckpointFilter( + name = Some("Test checkpoints 1"), + properties = Map("executionID" -> Set("id1", "id2")), + from = Some(ZonedDateTime.parse("2026-06-01T00:00:00Z")) + ) + val result = PartitioningReader(AtumPartitions(List("a" -> "b", "c" -> "d"))).getCheckpointsPage(filter = filter) + assert(result.map(_.data.size) == Right(2)) + } + test("The partitioning additional data are properly queried and delivered as DTO") { val atumPartitions: AtumPartitions = AtumPartitions( List( diff --git a/reader/src/test/scala/za/co/absa/atum/reader/requests/CheckpointFilterUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/requests/CheckpointFilterUnitTests.scala new file mode 100644 index 000000000..74ed24ae6 --- /dev/null +++ b/reader/src/test/scala/za/co/absa/atum/reader/requests/CheckpointFilterUnitTests.scala @@ -0,0 +1,73 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.requests + +import org.scalatest.funsuite.AnyFunSuiteLike +import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ + +import java.time.ZonedDateTime + +class CheckpointFilterUnitTests extends AnyFunSuiteLike { + + test("An empty filter adds no query params") { + assert(CheckpointFilter.empty.toQueryParams.isEmpty) + } + + test("Single-value properties are sent in the single-value format understood by older servers too") { + val filter = CheckpointFilter(properties = Map("executionID" -> Set("019f8981-7868-79fc-81d3-8143a4706f8a"))) + // base64url-encoded {"executionID":"019f8981-7868-79fc-81d3-8143a4706f8a"} + assert( + filter.toQueryParams == Map( + "checkpoint-properties" -> "eyJleGVjdXRpb25JRCI6IjAxOWY4OTgxLTc4NjgtNzlmYy04MWQzLTgxNDNhNDcwNmY4YSJ9" + ) + ) + } + + test("Multi-value properties are sent as arrays of values, all properties in the same format") { + val filter = CheckpointFilter(properties = Map("executionID" -> Set("b", "a"), "env" -> Set("prod"))) + val encoded = filter.toQueryParams("checkpoint-properties") + assert( + encoded.fromBase64As[Map[String, Seq[String]]] == Right(Map("executionID" -> Seq("a", "b"), "env" -> Seq("prod"))) + ) + } + + test("The time window is sent as UTC instants and all filter parts are combined") { + val filter = CheckpointFilter( + name = Some("checkpoint name"), + from = Some(ZonedDateTime.parse("2026-06-01T02:00:00+02:00[Europe/Budapest]")), + to = Some(ZonedDateTime.parse("2026-08-01T00:00:00.5Z")), + latestFirst = Some(false) + ) + assert( + filter.toQueryParams == Map( + "checkpoint-name" -> "checkpoint name", + "from" -> "2026-06-01T00:00:00Z", + "to" -> "2026-08-01T00:00:00.500Z", + "latest-first" -> "false" + ) + ) + } + + test("A property without any value is rejected, as it would match no checkpoint") { + val exception = intercept[IllegalArgumentException] { + CheckpointFilter(properties = Map("executionID" -> Set.empty[String], "env" -> Set("prod"))) + } + assert(exception.getMessage.contains("executionID")) + assert(!exception.getMessage.contains("env")) + } + +} diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala index 871e13622..b04dc1f8b 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpoints.scala @@ -27,10 +27,16 @@ import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstErrorStatusA import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling import zio._ import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbGet +import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbPut +import io.circe.syntax._ +import io.circe.generic.auto._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.implicits._ + import za.co.absa.atum.server.model.database.CheckpointItemWithPartitioningFromDB +import java.time.ZonedDateTime + class GetFlowCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieMultipleResultFunctionWithAggStatus[GetFlowCheckpointsArgs, Option[ CheckpointItemWithPartitioningFromDB @@ -40,7 +46,10 @@ class GetFlowCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task] fr"${input.limit}", fr"${input.offset}", fr"${input.checkpointName}", - fr"${input.checkpointProperties}" + fr"${input.checkpointProperties.map(_.asJson)}", + fr"${input.latestFirst}", + fr"${input.from}", + fr"${input.to}" ) ) with StandardStatusHandling @@ -69,7 +78,10 @@ object GetFlowCheckpoints { limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]] + checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime] ) val layer: URLayer[PostgresDatabaseProvider, GetFlowCheckpoints] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala index 7dd426144..0632ac744 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpoints.scala @@ -25,12 +25,18 @@ import za.co.absa.db.fadb.doobie.DoobieFunction.DoobieMultipleResultFunctionWith import zio._ import za.co.absa.atum.server.api.database.DoobieImplicits.Sequence.get import doobie.postgres.implicits._ + import za.co.absa.atum.server.api.database.runs.functions.GetPartitioningCheckpoints.GetPartitioningCheckpointsArgs import za.co.absa.atum.server.model.database.CheckpointItemFromDB import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbGet +import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbPut +import io.circe.syntax._ +import io.circe.generic.auto._ import za.co.absa.db.fadb.status.aggregation.implementations.ByFirstRowStatusAggregator import za.co.absa.db.fadb.status.handling.implementations.StandardStatusHandling +import java.time.ZonedDateTime + class GetPartitioningCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) extends DoobieMultipleResultFunctionWithAggStatus[GetPartitioningCheckpointsArgs, Option[ CheckpointItemFromDB.Paginated @@ -40,8 +46,10 @@ class GetPartitioningCheckpoints(implicit schema: DBSchema, dbEngine: DoobieEngi fr"${args.limit}", fr"${args.offset}", fr"${args.checkpointName}", - fr"${args.checkpointProperties}", - fr"${args.latestFirst}" + fr"${args.checkpointProperties.map(_.asJson)}", + fr"${args.latestFirst}", + fr"${args.from}", + fr"${args.to}" ) ) with StandardStatusHandling @@ -67,8 +75,10 @@ object GetPartitioningCheckpoints { limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], - latestFirst: Option[Boolean] + checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime] ) val layer: URLayer[PostgresDatabaseProvider, GetPartitioningCheckpoints] = ZLayer { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointController.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointController.scala index d5c790cae..4d521b999 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointController.scala @@ -22,6 +22,7 @@ import za.co.absa.atum.model.envelopes.SuccessResponse.{PaginatedResponse, Singl import zio.IO import zio.macros.accessible +import java.time.ZonedDateTime import java.util.UUID @accessible @@ -43,8 +44,10 @@ trait CheckpointController { limit: Int, offset: Long, checkpointName: Option[String] = None, - checkpointProperties: Option[Map[String, String]] = None, + checkpointProperties: Option[Map[String, Seq[String]]] = None, latestFirst: Option[Boolean] = None, + from: Option[ZonedDateTime] = None, + to: Option[ZonedDateTime] = None, includeProperties: Boolean ): IO[ErrorResponse, PaginatedResponse[CheckpointV2DTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerImpl.scala index 0a984b2ac..464cf05ec 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerImpl.scala @@ -25,6 +25,7 @@ import za.co.absa.atum.server.api.v2.service.CheckpointService import za.co.absa.atum.server.model.PaginatedResult import zio._ +import java.time.ZonedDateTime import java.util.UUID class CheckpointControllerImpl(checkpointService: CheckpointService) extends CheckpointController with BaseController { @@ -64,8 +65,10 @@ class CheckpointControllerImpl(checkpointService: CheckpointService) extends Che limit: Int, offset: Long, checkpointName: Option[String] = None, - checkpointProperties: Option[Map[String, String]] = None, + checkpointProperties: Option[Map[String, Seq[String]]] = None, latestFirst: Option[Boolean] = None, + from: Option[ZonedDateTime] = None, + to: Option[ZonedDateTime] = None, includeProperties: Boolean ): IO[ErrorResponse, PaginatedResponse[CheckpointV2DTO]] = { mapToPaginatedResponse( @@ -79,6 +82,8 @@ class CheckpointControllerImpl(checkpointService: CheckpointService) extends Che checkpointName, checkpointProperties, latestFirst, + from, + to, includeProperties ) ) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowController.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowController.scala index aed35322f..90cca4761 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowController.scala @@ -22,6 +22,8 @@ import za.co.absa.atum.model.envelopes.SuccessResponse.PaginatedResponse import zio.IO import zio.macros.accessible +import java.time.ZonedDateTime + @accessible trait FlowController { @@ -30,7 +32,10 @@ trait FlowController { limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ErrorResponse, PaginatedResponse[CheckpointWithPartitioningDTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerImpl.scala index 3dd824edb..3d1419de0 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerImpl.scala @@ -24,6 +24,8 @@ import za.co.absa.atum.server.api.v2.service.FlowService import za.co.absa.atum.server.model.PaginatedResult import zio._ +import java.time.ZonedDateTime + class FlowControllerImpl(flowService: FlowService) extends FlowController with BaseController { // to be replaced (and moved to checkpointcontroller) with new implementation in #233 @@ -32,12 +34,25 @@ class FlowControllerImpl(flowService: FlowService) extends FlowController with B limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ErrorResponse, PaginatedResponse[CheckpointWithPartitioningDTO]] = { val flowData = serviceCall[PaginatedResult[CheckpointWithPartitioningDTO], PaginatedResult[CheckpointWithPartitioningDTO]]( - flowService.getFlowCheckpoints(flowId, limit, offset, checkpointName, checkpointProperties, includeProperties) + flowService.getFlowCheckpoints( + flowId, + limit, + offset, + checkpointName, + checkpointProperties, + latestFirst, + from, + to, + includeProperties + ) ) mapToPaginatedResponse(limit, offset, flowData) } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala index 11934d6f5..34db55dec 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/http/Endpoints.scala @@ -20,7 +20,7 @@ import sttp.model.StatusCode import sttp.tapir.generic.auto.schemaForCaseClass import sttp.tapir.json.circe.jsonBody import sttp.tapir.ztapir._ -import sttp.tapir.{Codec, CodecFormat, DecodeResult, PublicEndpoint, Validator} +import sttp.tapir.{Codec, CodecFormat, DecodeResult, EndpointInput, PublicEndpoint, ValidationResult, Validator} import za.co.absa.atum.model.ApiPaths._ import za.co.absa.atum.model.dto._ import za.co.absa.atum.model.envelopes.ErrorResponse @@ -29,21 +29,50 @@ import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ import za.co.absa.atum.server.api.v2.controller.{CheckpointController, FlowController, PartitioningController} import za.co.absa.atum.server.api.common.http.{BaseEndpoints, HttpEnv} +import java.time.ZonedDateTime import java.util.UUID import scala.util.{Failure, Success, Try} object Endpoints extends BaseEndpoints { // Checkpoint properties are supplied as a single query parameter carrying a base64url-encoded JSON object, - // e.g. `checkpoint-properties=eyJleGVjdXRpb25JRCI6Ii4uLiJ9`. Malformed base64 or JSON results in a 400 Bad Request. - private implicit val checkpointPropertiesQueryCodec: Codec[String, Map[String, String], CodecFormat.TextPlain] = + // e.g. `checkpoint-properties=eyJleGVjdXRpb25JRCI6Ii4uLiJ9`. Malformed base64 or JSON, or a property without any + // accepted value (e.g. `{"executionID": []}`, which would match no checkpoint), results in a 400 Bad Request. + private implicit val checkpointPropertiesQueryCodec: Codec[String, Map[String, Seq[String]], CodecFormat.TextPlain] = Codec.string.mapDecode { encoded => - Try(encoded.fromBase64As[Map[String, String]]) match { + Try(encoded.fromBase64As[Map[String, Seq[String]]]) match { case Success(Right(properties)) => DecodeResult.Value(properties) - case Success(Left(error)) => DecodeResult.Error(encoded, error) - case Failure(exception) => DecodeResult.Error(encoded, exception) + case _ => + Try(encoded.fromBase64As[Map[String, String]]) match { + case Success(Right(properties)) => DecodeResult.Value(properties.map { case (k, v) => k -> Seq(v) }) + case Success(Left(error)) => DecodeResult.Error(encoded, error) + case Failure(exception) => DecodeResult.Error(encoded, exception) + } } }(_.asBase64EncodedJsonString) + .validate(Validator.custom { properties => + val propertiesWithoutValues = properties.filter(_._2.isEmpty).keys + if (propertiesWithoutValues.isEmpty) ValidationResult.Valid + else + ValidationResult.Invalid( + s"Checkpoint properties without any accepted value: ${propertiesWithoutValues.mkString(", ")}" + ) + }) + + // Optional half-open window `from <= process start time < to` for checkpoint queries, supplied as ISO-8601 date-times + // with an offset, e.g. `from=2026-06-01T00:00:00Z`. A malformed value, or `from` not before `to`, results in a + // 400 Bad Request. + private val checkpointTimeWindowInput: EndpointInput[(Option[ZonedDateTime], Option[ZonedDateTime])] = + query[Option[ZonedDateTime]]("from") + .description("inclusive lower bound of the checkpoint's process start time (ISO-8601 date-time with offset)") + .and( + query[Option[ZonedDateTime]]("to") + .description("exclusive upper bound of the checkpoint's process start time (ISO-8601 date-time with offset)") + ) + .validate(Validator.custom { + case (Some(from), Some(to)) if !from.isBefore(to) => ValidationResult.Invalid("'from' must be before 'to'") + case _ => ValidationResult.Valid + }) val postCheckpointEndpoint : PublicEndpoint[(Long, CheckpointV2DTO), ErrorResponse, (SingleSuccessResponse[CheckpointV2DTO], String), Any] = { @@ -113,7 +142,7 @@ object Endpoints extends BaseEndpoints { } val getPartitioningCheckpointsEndpoint - : PublicEndpoint[(Long, Int, Long, Option[String], Option[Map[String, String]], Option[Boolean], Boolean), ErrorResponse, PaginatedResponse[ + : PublicEndpoint[(Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Option[ZonedDateTime], Option[ZonedDateTime], Boolean), ErrorResponse, PaginatedResponse[ CheckpointV2DTO ], Any] = { apiV2.get @@ -122,8 +151,9 @@ object Endpoints extends BaseEndpoints { .in(query[Int]("limit").default(10).validate(Validator.inRange(1, 1000))) .in(query[Long]("offset").default(0L).validate(Validator.min(0L))) .in(query[Option[String]]("checkpoint-name")) - .in(query[Option[Map[String, String]]]("checkpoint-properties")) + .in(query[Option[Map[String, Seq[String]]]]("checkpoint-properties")) .in(query[Option[Boolean]]("latest-first")) + .in(checkpointTimeWindowInput) .in(query[Boolean]("include-properties").default(false)) .out(statusCode(StatusCode.Ok)) .out(jsonBody[PaginatedResponse[CheckpointV2DTO]]) @@ -131,7 +161,7 @@ object Endpoints extends BaseEndpoints { } val getFlowCheckpointsEndpoint - : PublicEndpoint[(Long, Int, Long, Option[String], Option[Map[String, String]], Boolean), ErrorResponse, PaginatedResponse[ + : PublicEndpoint[(Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Option[ZonedDateTime], Option[ZonedDateTime], Boolean), ErrorResponse, PaginatedResponse[ CheckpointWithPartitioningDTO ], Any] = { apiV2.get @@ -139,7 +169,9 @@ object Endpoints extends BaseEndpoints { .in(query[Int]("limit").default(10).validate(Validator.inRange(1, 1000))) .in(query[Long]("offset").default(0L).validate(Validator.min(0L))) .in(query[Option[String]]("checkpoint-name")) - .in(query[Option[Map[String, String]]]("checkpoint-properties")) + .in(query[Option[Map[String, Seq[String]]]]("checkpoint-properties")) + .in(query[Option[Boolean]]("latest-first")) + .in(checkpointTimeWindowInput) .in(query[Boolean]("include-properties").default(false)) .out(statusCode(StatusCode.Ok)) .out(jsonBody[PaginatedResponse[CheckpointWithPartitioningDTO]]) @@ -248,23 +280,23 @@ object Endpoints extends BaseEndpoints { } ), createServerEndpoint[ - (Long, Int, Long, Option[String], Option[Map[String, String]], Option[Boolean], Boolean), + (Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Option[ZonedDateTime], Option[ZonedDateTime], Boolean), ErrorResponse, PaginatedResponse[CheckpointV2DTO] ]( getPartitioningCheckpointsEndpoint, - { case (partitioningId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, String]], latestFirst: Option[Boolean], includeProperties: Boolean) => - CheckpointController.getPartitioningCheckpoints(partitioningId, limit, offset, checkpointName, checkpointProperties, latestFirst, includeProperties) + { case (partitioningId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], from: Option[ZonedDateTime], to: Option[ZonedDateTime], includeProperties: Boolean) => + CheckpointController.getPartitioningCheckpoints(partitioningId, limit, offset, checkpointName, checkpointProperties, latestFirst, from, to, includeProperties) } ), createServerEndpoint[ - (Long, Int, Long, Option[String], Option[Map[String, String]], Boolean), + (Long, Int, Long, Option[String], Option[Map[String, Seq[String]]], Option[Boolean], Option[ZonedDateTime], Option[ZonedDateTime], Boolean), ErrorResponse, PaginatedResponse[CheckpointWithPartitioningDTO] ]( getFlowCheckpointsEndpoint, - { case (flowId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, String]], includeProperties: Boolean) => - FlowController.getFlowCheckpoints(flowId, limit, offset, checkpointName, checkpointProperties, includeProperties) + { case (flowId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], from: Option[ZonedDateTime], to: Option[ZonedDateTime], includeProperties: Boolean) => + FlowController.getFlowCheckpoints(flowId, limit, offset, checkpointName, checkpointProperties, latestFirst, from, to, includeProperties) } ), createServerEndpoint(getPartitioningByIdEndpoint, PartitioningController.getPartitioningById), diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepository.scala index 1cf80366c..018cf3df6 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepository.scala @@ -22,6 +22,7 @@ import za.co.absa.atum.server.model.PaginatedResult import zio._ import zio.macros.accessible +import java.time.ZonedDateTime import java.util.UUID @accessible @@ -37,8 +38,10 @@ trait CheckpointRepository { limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[DatabaseError, PaginatedResult[CheckpointV2DTO]] } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryImpl.scala index 00dee0904..d1c13b425 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryImpl.scala @@ -31,6 +31,7 @@ import za.co.absa.atum.server.model.database.CheckpointItemFromDB import zio._ import zio.interop.catz.asyncInstance +import java.time.ZonedDateTime import java.util.UUID class CheckpointRepositoryImpl( @@ -75,13 +76,24 @@ class CheckpointRepositoryImpl( limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[DatabaseError, PaginatedResult[CheckpointV2DTO]] = { dbMultipleResultCallWithAggregatedStatus( getPartitioningCheckpointsFn( - GetPartitioningCheckpointsArgs(partitioningId, limit, offset, checkpointName, checkpointProperties, latestFirst) + GetPartitioningCheckpointsArgs( + partitioningId, + limit, + offset, + checkpointName, + checkpointProperties, + latestFirst, + from, + to + ) ), "getPartitioningCheckpoints" ) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepository.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepository.scala index f4095a38f..15e9b9951 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepository.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepository.scala @@ -22,6 +22,8 @@ import za.co.absa.atum.server.model.PaginatedResult import zio._ import zio.macros.accessible +import java.time.ZonedDateTime + @accessible trait FlowRepository { @@ -30,7 +32,10 @@ trait FlowRepository { limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[DatabaseError, PaginatedResult[CheckpointWithPartitioningDTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryImpl.scala index 94ec04d21..d861b046d 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryImpl.scala @@ -29,6 +29,8 @@ import za.co.absa.atum.server.model.database.CheckpointItemWithPartitioningFromD import zio._ import zio.interop.catz.asyncInstance +import java.time.ZonedDateTime + class FlowRepositoryImpl( getFlowCheckpointsFn: GetFlowCheckpoints, override val getCheckpointPropertiesFn: GetCheckpointProperties @@ -39,11 +41,16 @@ class FlowRepositoryImpl( limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[DatabaseError, PaginatedResult[CheckpointWithPartitioningDTO]] = { dbMultipleResultCallWithAggregatedStatus( - getFlowCheckpointsFn(GetFlowCheckpointsArgs(flowId, limit, offset, checkpointName, checkpointProperties)), + getFlowCheckpointsFn( + GetFlowCheckpointsArgs(flowId, limit, offset, checkpointName, checkpointProperties, latestFirst, from, to) + ), "getFlowCheckpoints" ) .map(_.flatten) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointService.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointService.scala index 9f2d77e7e..c59ca56cd 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointService.scala @@ -22,6 +22,7 @@ import za.co.absa.atum.server.model.PaginatedResult import zio.IO import zio.macros.accessible +import java.time.ZonedDateTime import java.util.UUID @accessible @@ -40,8 +41,10 @@ trait CheckpointService { limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ServiceError, PaginatedResult[CheckpointV2DTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceImpl.scala index 1eb434790..a53b57285 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceImpl.scala @@ -23,6 +23,7 @@ import za.co.absa.atum.server.api.v2.repository.CheckpointRepository import za.co.absa.atum.server.model.PaginatedResult import zio._ +import java.time.ZonedDateTime import java.util.UUID class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends CheckpointService with BaseService { @@ -50,8 +51,10 @@ class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ServiceError, PaginatedResult[CheckpointV2DTO]] = { repositoryCall( @@ -62,6 +65,8 @@ class CheckpointServiceImpl(checkpointRepository: CheckpointRepository) extends checkpointName, checkpointProperties, latestFirst, + from, + to, includeProperties ), "getPartitioningCheckpoints" diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowService.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowService.scala index 21a1c6101..5563086b9 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowService.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowService.scala @@ -22,6 +22,8 @@ import za.co.absa.atum.server.model.PaginatedResult import zio._ import zio.macros.accessible +import java.time.ZonedDateTime + @accessible trait FlowService { @@ -30,7 +32,10 @@ trait FlowService { limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ServiceError, PaginatedResult[CheckpointWithPartitioningDTO]] diff --git a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowServiceImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowServiceImpl.scala index d18db368d..555b23271 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowServiceImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/v2/service/FlowServiceImpl.scala @@ -23,6 +23,8 @@ import za.co.absa.atum.server.api.v2.repository.FlowRepository import za.co.absa.atum.server.model.PaginatedResult import zio._ +import java.time.ZonedDateTime + class FlowServiceImpl(flowRepository: FlowRepository) extends FlowService with BaseService { override def getFlowCheckpoints( @@ -30,11 +32,24 @@ class FlowServiceImpl(flowRepository: FlowRepository) extends FlowService with B limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ): IO[ServiceError, PaginatedResult[CheckpointWithPartitioningDTO]] = { repositoryCall( - flowRepository.getFlowCheckpoints(flowId, limit, offset, checkpointName, checkpointProperties, includeProperties), + flowRepository.getFlowCheckpoints( + flowId, + limit, + offset, + checkpointName, + checkpointProperties, + latestFirst, + from, + to, + includeProperties + ), "getFlowCheckpoints" ) } diff --git a/server/src/main/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDB.scala index 25b1cf9e7..2ad4d5337 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDB.scala @@ -98,10 +98,8 @@ object CheckpointItemWithPartitioningFromDB { checkpointItems: Seq[CheckpointItemWithPartitioningFromDB] ): Either[Throwable, Seq[CheckpointWithPartitioningDTO]] = { val groupedItems = checkpointItems.groupBy(_.idCheckpoint) - val orderedCheckpointIds = checkpointItems - .sortBy(_.checkpointStartTime)(Ordering[ZonedDateTime].reverse) - .map(_.idCheckpoint) - .distinct + // the DB function returns the items already ordered (latest or earliest first, as requested), so keep that order + val orderedCheckpointIds = checkpointItems.map(_.idCheckpoint).distinct val result = orderedCheckpointIds.map { id => fromItemsToCheckpointWithPartitioningDTO(groupedItems(id)) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala b/server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala index 08df37af7..39ade40d8 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala @@ -22,6 +22,8 @@ import za.co.absa.atum.server.config.PostgresConfig import zio._ import zio.interop.catz._ +import java.sql.DriverManager + object TestTransactorProvider { val layerWithoutRollback: ZLayer[Any, Config.Error, Transactor[Task]] = ZLayer { @@ -53,4 +55,27 @@ object TestTransactorProvider { } yield transactorWithRollback } + // All database calls share a single connection and a single transaction, rolled back once the layer is released. + // + // Why: some tests first insert test data (e.g. via CreatePartitioning, WriteCheckpointV2) and then read it back + // with the function under test (e.g. GetPartitioningCheckpoints), so the inserted rows must survive between calls: + // - layerWithRollback rolls back after each call, so the rows are gone before they can be read; + // - layerWithoutRollback keeps the rows, but permanently. + val layerWithSingleTransactionRollback: ZLayer[Any, Throwable, Transactor[Task]] = ZLayer.scoped { + for { + postgresConfig <- ZIO.config[PostgresConfig](PostgresConfig.config) + connection <- ZIO.acquireRelease( + ZIO.attemptBlocking { + val connection = DriverManager.getConnection( + s"jdbc:postgresql://${postgresConfig.serverName}:${postgresConfig.portNumber}/${postgresConfig.databaseName}", + postgresConfig.user, + postgresConfig.password + ) + connection.setAutoCommit(false) + connection + } + )(connection => ZIO.attemptBlocking(try connection.rollback() finally connection.close()).orDie) + } yield Transactor.strategy.set(Transactor.fromConnection[Task](connection), Strategy.void) + } + } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala index 326d5e20d..fda0905e4 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowCheckpointsIntegrationTests.scala @@ -16,39 +16,159 @@ package za.co.absa.atum.server.api.database.flows.functions +import za.co.absa.atum.model.ResultValueType +import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue +import za.co.absa.atum.model.dto._ import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider +import za.co.absa.atum.server.api.database.flows.functions.GetFlowCheckpoints.GetFlowCheckpointsArgs +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointArgs +import za.co.absa.atum.server.api.database.runs.functions.{ + CreatePartitioning, + GetPartitioningMainFlow, + WriteCheckpointV2 +} import za.co.absa.db.fadb.exceptions.DataNotFoundException import za.co.absa.db.fadb.status.FunctionStatus import zio._ import zio.test._ import zio.interop.catz.asyncInstance +import java.time.ZonedDateTime +import java.util.UUID + object GetFlowCheckpointsIntegrationTests extends ConfigProviderTest { + private val checkpointName = "Data written" + private val firstStartTime = ZonedDateTime.parse("2026-01-01T00:00:00Z") + + private def createPartitioning: ZIO[CreatePartitioning, Throwable, Long] = + ZIO + .serviceWithZIO[CreatePartitioning]( + _(PartitioningSubmitV2DTO(Seq(PartitionDTO("key", UUID.randomUUID().toString)), None, "author")) + ) + .flatMap(ZIO.fromEither(_)) + .map(_.data) + + private def getMainFlowId(partitioningId: Long): ZIO[GetPartitioningMainFlow, Throwable, Long] = + ZIO + .serviceWithZIO[GetPartitioningMainFlow](_(partitioningId)) + .flatMap(ZIO.fromEither(_)) + .flatMap(row => ZIO.getOrFail(row.data.map(_.id))) + + private def writeCheckpoint( + partitioningId: Long, + name: String, + executionId: Option[String], + startMinute: Long + ): ZIO[WriteCheckpointV2, Throwable, UUID] = { + val checkpoint = CheckpointV2DTO( + id = UUID.randomUUID(), + name = name, + author = "author", + processStartTime = firstStartTime.plusMinutes(startMinute), + processEndTime = None, + measurements = Set( + MeasurementDTO(MeasureDTO("count", Seq("*")), MeasureResultDTO(TypedValue("1", ResultValueType.LongValue))) + ), + properties = executionId.map(id => Map("executionID" -> id)) + ) + ZIO + .serviceWithZIO[WriteCheckpointV2](_(WriteCheckpointArgs(partitioningId, checkpoint))) + .flatMap(ZIO.fromEither(_)) + .as(checkpoint.id) + } + + // returns the IDs of the checkpoints on the page, in the order received, and the has-more flag + private def getPage(args: GetFlowCheckpointsArgs): ZIO[GetFlowCheckpoints, Throwable, (Seq[UUID], Boolean)] = + ZIO.serviceWithZIO[GetFlowCheckpoints](_(args)).flatMap(ZIO.fromEither(_)).map { rows => + val items = rows.flatMap(_.data) + (items.map(_.idCheckpoint), items.exists(_.hasMore)) + } + override def spec: Spec[TestEnvironment with Scope, Any] = { suite("GetFlowCheckpointsIntegrationTests")( - test("Should return checkpoints with the correct flowId, limit, and offset") { + test("Should return DataNotFoundException when flow does not exist") { - val args = GetFlowCheckpoints.GetFlowCheckpointsArgs( + val args = GetFlowCheckpointsArgs( flowId = 1L, limit = 10, offset = 0L, checkpointName = Some("TestCheckpointName"), - checkpointProperties = None + checkpointProperties = None, + latestFirst = None, + from = None, + to = None ) for { getFlowCheckpoints <- ZIO.service[GetFlowCheckpoints] result <- getFlowCheckpoints(args) } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(42, "Flow not found")))) + }, + test("Should apply pagination (limit and offset) accurately with combined filters") { + def getFilteredPage(flowId: Long, offset: Long) = getPage( + GetFlowCheckpointsArgs( + flowId = flowId, + limit = 2, + offset = offset, + checkpointName = Some(checkpointName), + checkpointProperties = Some(Map("executionID" -> Seq("a", "b"))), + latestFirst = None, + from = None, + to = None + ) + ) + + for { + partitioningId <- createPartitioning + flowId <- getMainFlowId(partitioningId) + matching1 <- writeCheckpoint(partitioningId, checkpointName, Some("a"), startMinute = 1) + matching2 <- writeCheckpoint(partitioningId, checkpointName, Some("b"), startMinute = 2) + _ <- writeCheckpoint(partitioningId, checkpointName, Some("c"), startMinute = 3) + _ <- writeCheckpoint(partitioningId, "Other checkpoint", Some("a"), startMinute = 4) + _ <- writeCheckpoint(partitioningId, checkpointName, None, startMinute = 5) + matching3 <- writeCheckpoint(partitioningId, checkpointName, Some("a"), startMinute = 6) + firstPage <- getFilteredPage(flowId, offset = 0L) + secondPage <- getFilteredPage(flowId, offset = 2L) + } yield assertTrue( + firstPage == (Seq(matching3, matching2), true), + secondPage == (Seq(matching1), false) + ) + }, + test("Should apply the earliest-first order and the process start time window") { + def getWindowPage(flowId: Long) = getPage( + GetFlowCheckpointsArgs( + flowId = flowId, + limit = 10, + offset = 0L, + checkpointName = None, + checkpointProperties = None, + latestFirst = Some(false), + from = Some(firstStartTime.plusMinutes(2)), + to = Some(firstStartTime.plusMinutes(4)) + ) + ) + + for { + partitioningId <- createPartitioning + flowId <- getMainFlowId(partitioningId) + _ <- writeCheckpoint(partitioningId, checkpointName, None, startMinute = 1) + atWindowStart <- writeCheckpoint(partitioningId, checkpointName, None, startMinute = 2) + inWindow <- writeCheckpoint(partitioningId, checkpointName, None, startMinute = 3) + _ <- writeCheckpoint(partitioningId, checkpointName, None, startMinute = 4) + page <- getWindowPage(flowId) + } yield assertTrue(page == (Seq(atWindowStart, inWindow), false)) } ).provide( GetFlowCheckpoints.layer, + CreatePartitioning.layer, + GetPartitioningMainFlow.layer, + WriteCheckpointV2.layer, PostgresDatabaseProvider.layer, - TestTransactorProvider.layerWithRollback + TestTransactorProvider.layerWithSingleTransactionRollback ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowPartitioningsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowPartitioningsIntegrationTests.scala index af3a3232c..11bed2633 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowPartitioningsIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/flows/functions/GetFlowPartitioningsIntegrationTests.scala @@ -16,18 +16,37 @@ package za.co.absa.atum.server.api.database.flows.functions +import za.co.absa.atum.model.dto.{PartitionDTO, PartitioningSubmitV2DTO} import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.flows.functions.GetFlowPartitionings.GetFlowPartitioningsArgs +import za.co.absa.atum.server.api.database.runs.functions.{CreatePartitioning, GetPartitioningMainFlow} import za.co.absa.db.fadb.exceptions.DataNotFoundException import za.co.absa.db.fadb.status.FunctionStatus import zio.{Scope, ZIO} import zio.test.{Spec, TestEnvironment, assertTrue} import zio.interop.catz.asyncInstance +import java.util.UUID + object GetFlowPartitioningsIntegrationTests extends ConfigProviderTest { + private def createPartitioning(parentId: Option[Long]): ZIO[CreatePartitioning, Throwable, Long] = + ZIO + .serviceWithZIO[CreatePartitioning]( + _(PartitioningSubmitV2DTO(Seq(PartitionDTO("page", UUID.randomUUID().toString)), parentId, "Pagination author")) + ) + .flatMap(ZIO.fromEither(_)) + .map(_.data) + + private def getPage(flowId: Long, offset: Long): ZIO[GetFlowPartitionings, Throwable, Seq[(Long, Boolean)]] = + ZIO + .serviceWithZIO[GetFlowPartitionings](_(GetFlowPartitioningsArgs(flowId, 2, offset))) + .flatMap(ZIO.fromEither(_)) + .flatMap(rows => ZIO.foreach(rows)(row => ZIO.getOrFail(row.data))) + .map(_.map(row => (row.id, row.hasMore))) + override def spec: Spec[Unit with TestEnvironment with Scope, Any] = { suite("GetFlowPartitioningsIntegrationTests")( test("Returns expected DataNotFoundException when flow not found") { @@ -35,12 +54,34 @@ object GetFlowPartitioningsIntegrationTests extends ConfigProviderTest { getFlowPartitionings <- ZIO.service[GetFlowPartitionings] result <- getFlowPartitionings(GetFlowPartitioningsArgs(0L, 1, 1L)) } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(41, "Flow not found")))) + }, + test("Decodes ordered partitionings and paging metadata at positive offsets") { + for { + parentId <- createPartitioning(None) + childIds <- ZIO.foreach(1 to 4)(_ => createPartitioning(Some(parentId))) + ids = (parentId +: childIds).sorted + mainFlow <- ZIO.serviceWithZIO[GetPartitioningMainFlow](_(parentId)).flatMap(ZIO.fromEither(_)) + flowId <- ZIO.getOrFail(mainFlow.data.map(_.id)) + firstPage <- getPage(flowId, 0L) + middlePage <- getPage(flowId, 2L) + lastPartialPage <- getPage(flowId, 4L) + lastFullPage <- getPage(flowId, 3L) + beyondEndPage <- getPage(flowId, 6L) + } yield assertTrue( + firstPage == ids.take(2).map(id => (id, true)), + middlePage == ids.slice(2, 4).map(id => (id, true)), + lastPartialPage == Seq((ids.last, false)), + lastFullPage == ids.takeRight(2).map(id => (id, false)), + beyondEndPage.isEmpty + ) } ) }.provide( GetFlowPartitionings.layer, + CreatePartitioning.layer, + GetPartitioningMainFlow.layer, PostgresDatabaseProvider.layer, - TestTransactorProvider.layerWithRollback + TestTransactorProvider.layerWithSingleTransactionRollback ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala index a0aad54a1..db1d04999 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/database/runs/functions/GetPartitioningCheckpointsIntegrationTests.scala @@ -16,30 +16,148 @@ package za.co.absa.atum.server.api.database.runs.functions +import za.co.absa.atum.model.ResultValueType +import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue +import za.co.absa.atum.model.dto._ import za.co.absa.atum.server.ConfigProviderTest import za.co.absa.atum.server.api.TestTransactorProvider import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.functions.GetPartitioningCheckpoints.GetPartitioningCheckpointsArgs +import za.co.absa.atum.server.api.database.runs.functions.WriteCheckpointV2.WriteCheckpointArgs import za.co.absa.db.fadb.exceptions.DataNotFoundException import za.co.absa.db.fadb.status.FunctionStatus import zio.interop.catz.asyncInstance import zio.{Scope, ZIO} import zio.test._ +import java.time.ZonedDateTime +import java.util.UUID + object GetPartitioningCheckpointsIntegrationTests extends ConfigProviderTest { + private val checkpointName = "Data written" + private val firstStartTime = ZonedDateTime.parse("2026-01-01T00:00:00Z") + + private def createPartitioning: ZIO[CreatePartitioning, Throwable, Long] = + ZIO + .serviceWithZIO[CreatePartitioning]( + _(PartitioningSubmitV2DTO(Seq(PartitionDTO("key", UUID.randomUUID().toString)), None, "author")) + ) + .flatMap(ZIO.fromEither(_)) + .map(_.data) + + private def writeCheckpoint( + partitioningId: Long, + name: String, + executionId: Option[String], + startMinute: Long + ): ZIO[WriteCheckpointV2, Throwable, UUID] = { + val checkpoint = CheckpointV2DTO( + id = UUID.randomUUID(), + name = name, + author = "author", + processStartTime = firstStartTime.plusMinutes(startMinute), + processEndTime = None, + measurements = Set( + MeasurementDTO(MeasureDTO("count", Seq("*")), MeasureResultDTO(TypedValue("1", ResultValueType.LongValue))) + ), + properties = executionId.map(id => Map("executionID" -> id)) + ) + ZIO + .serviceWithZIO[WriteCheckpointV2](_(WriteCheckpointArgs(partitioningId, checkpoint))) + .flatMap(ZIO.fromEither(_)) + .as(checkpoint.id) + } + + // returns the IDs of the checkpoints on the page, in the order received, and the has-more flag + private def getPage( + args: GetPartitioningCheckpointsArgs + ): ZIO[GetPartitioningCheckpoints, Throwable, (Seq[UUID], Boolean)] = + ZIO.serviceWithZIO[GetPartitioningCheckpoints](_(args)).flatMap(ZIO.fromEither(_)).map { rows => + val items = rows.flatMap(_.data) + (items.map(_.idCheckpoint), items.exists(_.hasMore)) + } + override def spec: Spec[TestEnvironment with Scope, Any] = { suite("GetPartitioningCheckpointsIntegrationTests")( - test("Returns expected sequence of Checkpoints with non-existing partitioning id") { + test("Should return DataNotFoundException when partitioning does not exist") { + val args = GetPartitioningCheckpointsArgs( + partitioningId = 0L, + limit = 10, + offset = 0L, + checkpointName = None, + checkpointProperties = None, + latestFirst = None, + from = None, + to = None + ) + for { getPartitioningCheckpoints <- ZIO.service[GetPartitioningCheckpoints] - result <- getPartitioningCheckpoints(GetPartitioningCheckpointsArgs(0L, 10, 0L, None, None, None)) + result <- getPartitioningCheckpoints(args) } yield assertTrue(result == Left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) + }, + test("Should apply pagination (limit and offset) accurately with combined filters") { + def getFilteredPage(partitioningId: Long, offset: Long, latestFirst: Option[Boolean] = None) = getPage( + GetPartitioningCheckpointsArgs( + partitioningId = partitioningId, + limit = 2, + offset = offset, + checkpointName = Some(checkpointName), + checkpointProperties = Some(Map("executionID" -> Seq("a", "b"))), + latestFirst = latestFirst, + from = None, + to = None + + ) + ) + + for { + partitioningId <- createPartitioning + matching1 <- writeCheckpoint(partitioningId, checkpointName, Some("a"), startMinute = 1) + matching2 <- writeCheckpoint(partitioningId, checkpointName, Some("b"), startMinute = 2) + _ <- writeCheckpoint(partitioningId, checkpointName, Some("c"), startMinute = 3) + _ <- writeCheckpoint(partitioningId, "Other checkpoint", Some("a"), startMinute = 4) + _ <- writeCheckpoint(partitioningId, checkpointName, None, startMinute = 5) + matching3 <- writeCheckpoint(partitioningId, checkpointName, Some("a"), startMinute = 6) + firstPage <- getFilteredPage(partitioningId, offset = 0L) + secondPage <- getFilteredPage(partitioningId, offset = 2L) + earliestFirstPage <- getFilteredPage(partitioningId, offset = 0L, latestFirst = Some(false)) + } yield assertTrue( + firstPage == (Seq(matching3, matching2), true), + secondPage == (Seq(matching1), false), + earliestFirstPage == (Seq(matching1, matching2), true) + ) + }, + test("Should apply the earliest-first order and the process start time window") { + def getWindowPage(partitioningId: Long) = getPage( + GetPartitioningCheckpointsArgs( + partitioningId = partitioningId, + limit = 10, + offset = 0L, + checkpointName = None, + checkpointProperties = None, + latestFirst = Some(false), + from = Some(firstStartTime.plusMinutes(2)), + to = Some(firstStartTime.plusMinutes(4)) + ) + ) + + for { + partitioningId <- createPartitioning + _ <- writeCheckpoint(partitioningId, checkpointName, None, startMinute = 1) + atWindowStart <- writeCheckpoint(partitioningId, checkpointName, None, startMinute = 2) + inWindow <- writeCheckpoint(partitioningId, checkpointName, None, startMinute = 3) + _ <- writeCheckpoint(partitioningId, checkpointName, None, startMinute = 4) + page <- getWindowPage(partitioningId) + } yield assertTrue(page == (Seq(atWindowStart, inWindow), false)) } ).provide( GetPartitioningCheckpoints.layer, + CreatePartitioning.layer, + WriteCheckpointV2.layer, PostgresDatabaseProvider.layer, - TestTransactorProvider.layerWithRollback + TestTransactorProvider.layerWithSingleTransactionRollback ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerUnitTests.scala index 9a846d9d5..b44674f70 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/CheckpointControllerUnitTests.scala @@ -51,11 +51,11 @@ object CheckpointControllerUnitTests extends ConfigProviderTest with TestData { when(checkpointServiceMock.getCheckpoint(partitioningId1, checkpointV2DTO3.id, includeProperties = false)) .thenReturn(ZIO.fail(GeneralServiceError("boom!"))) - when(checkpointServiceMock.getPartitioningCheckpoints(partitioningId1, 10, 0L, None, None, None, includeProperties = false)) + when(checkpointServiceMock.getPartitioningCheckpoints(partitioningId1, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultHasMore(Seq(checkpointV2DTO1)))) - when(checkpointServiceMock.getPartitioningCheckpoints(partitioningId2, 10, 0L, None, None, None, includeProperties = false)) + when(checkpointServiceMock.getPartitioningCheckpoints(partitioningId2, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultNoMore(Seq(checkpointV2DTO1)))) - when(checkpointServiceMock.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, includeProperties = false)) + when(checkpointServiceMock.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundServiceError("Partitioning not found"))) private val checkpointServiceMockLayer = ZLayer.succeed(checkpointServiceMock) diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerUnitTests.scala index d78be581e..eabe38774 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/controller/FlowControllerUnitTests.scala @@ -29,11 +29,11 @@ import zio.test._ object FlowControllerUnitTests extends ZIOSpecDefault with TestData { private val flowServiceMock = mock(classOf[FlowService]) - when(flowServiceMock.getFlowCheckpoints(1L, 5, 2L, None, None, includeProperties = false)) + when(flowServiceMock.getFlowCheckpoints(1L, 5, 2L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultHasMore(Seq(checkpointWithPartitioningDTO1)))) - when(flowServiceMock.getFlowCheckpoints(2L, 5, 0L, None, None, includeProperties = false)) + when(flowServiceMock.getFlowCheckpoints(2L, 5, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultNoMore(Seq(checkpointWithPartitioningDTO2)))) - when(flowServiceMock.getFlowCheckpoints(3L, 5, 0L, None, None, includeProperties = false)) + when(flowServiceMock.getFlowCheckpoints(3L, 5, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundServiceError("Flow not found"))) private val flowServiceMockLayer = ZLayer.succeed(flowServiceMock) @@ -43,20 +43,20 @@ object FlowControllerUnitTests extends ZIOSpecDefault with TestData { suite("GetFlowCheckpointsV2Suite")( test("Returns expected Seq[CheckpointV2DTO] with Pagination indicating there is more data available") { for { - result <- FlowController.getFlowCheckpoints(1L, 5, 2L, None, None, includeProperties = false) + result <- FlowController.getFlowCheckpoints(1L, 5, 2L, None, None, None, None, None, includeProperties = false) } yield assertTrue( result.data == Seq(checkpointWithPartitioningDTO1) && result.pagination == Pagination(5, 2, hasMore = true) ) }, test("Returns expected Seq[CheckpointV2DTO] with Pagination indicating there is no more data available") { for { - result <- FlowController.getFlowCheckpoints(2L, 5, 0L, None, None, includeProperties = false) + result <- FlowController.getFlowCheckpoints(2L, 5, 0L, None, None, None, None, None, includeProperties = false) } yield assertTrue( result.data == Seq(checkpointWithPartitioningDTO2) && result.pagination == Pagination(5, 0, hasMore = false) ) }, test("Returns expected NotFoundServiceError when service returns NotFoundServiceError") { - assertZIO(FlowController.getFlowCheckpoints(3L, 5, 0L, None, None, includeProperties = false).exit)( + assertZIO(FlowController.getFlowCheckpoints(3L, 5, 0L, None, None, None, None, None, includeProperties = false).exit)( failsWithA[NotFoundErrorResponse] ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala index 364f2d676..fc5ba448e 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetFlowCheckpointsEndpointUnitTests.scala @@ -26,12 +26,14 @@ import sttp.tapir.ztapir.{RIOMonadError, RichZEndpoint} import za.co.absa.atum.model.dto.CheckpointWithPartitioningDTO import za.co.absa.atum.model.envelopes.SuccessResponse.PaginatedResponse import za.co.absa.atum.model.envelopes.{NotFoundErrorResponse, Pagination} +import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.v2.controller.FlowController import zio.test.Assertion.equalTo import zio.test.{Spec, TestEnvironment, ZIOSpecDefault, assertZIO} import zio.{Scope, ZIO, ZLayer} +import java.time.ZonedDateTime import java.util.UUID object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData { @@ -45,11 +47,11 @@ object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData private val encodedExecutionIdProperties = "eyJleGVjdXRpb25JRCI6IjAxOWY4OTgxLTc4NjgtNzlmYy04MWQzLTgxNDNhNDcwNmY4YSJ9" - when(flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, None, includeProperties = false)) + when(flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn( ZIO.succeed(PaginatedResponse(Seq(checkpointWithPartitioningDTO1), Pagination(5, 0, hasMore = true), uuid)) ) - when(flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, None, includeProperties = true)) + when(flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, None, None, None, None, includeProperties = true)) .thenReturn( ZIO.succeed( PaginatedResponse( @@ -59,24 +61,56 @@ object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData ) ) ) - when(flowControllerMockV2.getFlowCheckpoints(2L, 5, 0L, None, None, includeProperties = false)) + when(flowControllerMockV2.getFlowCheckpoints(2L, 5, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn( ZIO.succeed(PaginatedResponse(Seq(checkpointWithPartitioningDTO2), Pagination(5, 0, hasMore = false), uuid)) ) - when(flowControllerMockV2.getFlowCheckpoints(3L, 5, 0L, None, None, includeProperties = false)) + when(flowControllerMockV2.getFlowCheckpoints(3L, 5, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundErrorResponse("Flow not found for a given ID"))) when( - flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, Some(executionIdProperties), includeProperties = false) + flowControllerMockV2.getFlowCheckpoints(1L, 5, 0L, None, Some(executionIdProperties.view.mapValues(Seq(_)).toMap), None, None, None, includeProperties = false) ) .thenReturn( ZIO.succeed(PaginatedResponse(Seq(checkpointWithPartitioningDTO1), Pagination(5, 0, hasMore = true), uuid)) ) + private val windowFrom = ZonedDateTime.parse("2026-06-01T00:00:00Z") + private val windowTo = ZonedDateTime.parse("2026-08-01T00:00:00Z") + + when( + flowControllerMockV2.getFlowCheckpoints( + 1L, 5, 0L, None, None, Some(false), Some(windowFrom), Some(windowTo), includeProperties = false + ) + ) + .thenReturn( + ZIO.succeed(PaginatedResponse(Seq(checkpointWithPartitioningDTO2), Pagination(5, 0, hasMore = false), uuid)) + ) + private val flowControllerMockLayerV2 = ZLayer.succeed(flowControllerMockV2) private val getFlowCheckpointServerEndpoint = Endpoints.getFlowCheckpointsEndpoint.zServerLogic({ - case (flowId: Long, limit: Int, offset: Long, checkpointName: Option[String], checkpointProperties: Option[Map[String, String]], includeProperties: Boolean) => - FlowController.getFlowCheckpoints(flowId, limit, offset, checkpointName, checkpointProperties, includeProperties) + case ( + flowId: Long, + limit: Int, + offset: Long, + checkpointName: Option[String], + checkpointProperties: Option[Map[String, Seq[String]]], + latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], + includeProperties: Boolean + ) => + FlowController.getFlowCheckpoints( + flowId, + limit, + offset, + checkpointName, + checkpointProperties, + latestFirst, + from, + to, + includeProperties + ) }) def spec: Spec[TestEnvironment with Scope, Any] = { @@ -174,6 +208,47 @@ object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData ) ) }, + test("Returns an expected PaginatedResponse[CheckpointWithPartitioningDTO] when filtering by a time window, earliest first") { + val baseUri = + uri"https://test.com/api/v2/flows/1/checkpoints?limit=5&offset=0&latest-first=false&from=2026-06-01T00:00:00Z&to=2026-08-01T00:00:00Z" + val response = basicRequest + .get(baseUri) + .response(asJson[PaginatedResponse[CheckpointWithPartitioningDTO]]) + .send(backendStub) + + val body = response.map(_.body) + val statusCode = response.map(_.code) + + assertZIO(body <&> statusCode)( + equalTo( + Right(PaginatedResponse(Seq(checkpointWithPartitioningDTO2), Pagination(5, 0, hasMore = false), uuid)), + StatusCode.Ok + ) + ) + }, + test("Returns expected 400 when 'to' is not a valid date-time") { + val baseUri = uri"https://test.com/api/v2/flows/1/checkpoints?limit=5&offset=0&to=not-a-date-time" + val response = basicRequest + .get(baseUri) + .response(asJson[PaginatedResponse[CheckpointWithPartitioningDTO]]) + .send(backendStub) + + val statusCode = response.map(_.code) + + assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) + }, + test("Returns expected 400 when 'from' is after 'to'") { + val baseUri = + uri"https://test.com/api/v2/flows/1/checkpoints?limit=5&offset=0&from=2026-08-01T00:00:00Z&to=2026-06-01T00:00:00Z" + val response = basicRequest + .get(baseUri) + .response(asJson[PaginatedResponse[CheckpointWithPartitioningDTO]]) + .send(backendStub) + + val statusCode = response.map(_.code) + + assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) + }, test("Returns expected 400 when checkpoint-properties is not valid base64") { val baseUri = uri"https://test.com/api/v2/flows/1/checkpoints?limit=5&offset=0&checkpoint-properties=!!!not-base64!!!" @@ -186,6 +261,19 @@ object GetFlowCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) }, + test("Returns expected 400 when a checkpoint property has no accepted value") { + val encodedProps = Map("executionID" -> Seq.empty[String]).asBase64EncodedJsonString + val baseUri = + uri"https://test.com/api/v2/flows/1/checkpoints?limit=5&offset=0&checkpoint-properties=$encodedProps" + val response = basicRequest + .get(baseUri) + .response(asJson[PaginatedResponse[CheckpointWithPartitioningDTO]]) + .send(backendStub) + + val statusCode = response.map(_.code) + + assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) + }, test("Returns expected 400 when limit is out of range") { val baseUri = uri"https://test.com/api/v2/flows/1/checkpoints?limit=1005&offset=0" val response = basicRequest diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala index 5bef2b380..630089907 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/http/GetPartitioningCheckpointsEndpointUnitTests.scala @@ -26,12 +26,14 @@ import sttp.tapir.ztapir.{RIOMonadError, RichZEndpoint} import za.co.absa.atum.model.dto.CheckpointV2DTO import za.co.absa.atum.model.envelopes.SuccessResponse.PaginatedResponse import za.co.absa.atum.model.envelopes.{NotFoundErrorResponse, Pagination} +import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ import za.co.absa.atum.server.api.TestData import za.co.absa.atum.server.api.v2.controller.CheckpointController import zio.test.Assertion.equalTo import zio.test.{Spec, TestEnvironment, ZIOSpecDefault, assertZIO} import zio.{Scope, ZIO, ZLayer} +import java.time.ZonedDateTime import java.util.UUID object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with TestData { @@ -40,9 +42,12 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T private val uuid = UUID.randomUUID() - when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, includeProperties = false)) + private val windowFrom = ZonedDateTime.parse("2026-06-01T00:00:00Z") + private val windowTo = ZonedDateTime.parse("2026-08-01T00:00:00Z") + + when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(PaginatedResponse(Seq(checkpointV2DTO1), Pagination(10, 0, hasMore = true), uuid))) - when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, includeProperties = true)) + when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, None, None, includeProperties = true)) .thenReturn( ZIO.succeed( PaginatedResponse( @@ -52,12 +57,21 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T ) ) ) - when(checkpointControllerMock.getPartitioningCheckpoints(1L, 20, 0L, None, None, None, includeProperties = false)) + when(checkpointControllerMock.getPartitioningCheckpoints(1L, 20, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(PaginatedResponse(Seq(checkpointV2DTO1), Pagination(20, 0, hasMore = false), uuid))) - when(checkpointControllerMock.getPartitioningCheckpoints(2L, 10, 0L, None, None, None, includeProperties = false)) + when(checkpointControllerMock.getPartitioningCheckpoints(2L, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundErrorResponse("partitioning not found"))) - when(checkpointControllerMock.getPartitioningCheckpoints(3L, 10, 0L, None, None, None, includeProperties = false)) + when(checkpointControllerMock.getPartitioningCheckpoints(3L, 10, 0L, None, None, None, None, None, includeProperties = false)) + .thenReturn(ZIO.succeed(PaginatedResponse(Seq(checkpointV2DTO1), Pagination(10, 0, hasMore = true), uuid))) + when(checkpointControllerMock.getPartitioningCheckpoints(1L, 10, 0L, None, Some(Map("executionID" -> Seq("id1", "id2"))), None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(PaginatedResponse(Seq(checkpointV2DTO1), Pagination(10, 0, hasMore = true), uuid))) + when( + checkpointControllerMock.getPartitioningCheckpoints( + 1L, 10, 0L, None, None, None, Some(windowFrom), Some(windowTo), includeProperties = false + ) + ) + .thenReturn(ZIO.succeed(PaginatedResponse(Seq(checkpointV2DTO2), Pagination(10, 0, hasMore = false), uuid))) + private val checkpointControllerMockLayer = ZLayer.succeed(checkpointControllerMock) @@ -68,8 +82,10 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T limit: Int, offset: Long, checkpointName: Option[String], - checkpointProperties: Option[Map[String, String]], + checkpointProperties: Option[Map[String, Seq[String]]], latestFirst: Option[Boolean], + from: Option[ZonedDateTime], + to: Option[ZonedDateTime], includeProperties: Boolean ) => CheckpointController.getPartitioningCheckpoints( @@ -79,6 +95,8 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T checkpointName, checkpointProperties, latestFirst, + from, + to, includeProperties ) }) @@ -91,6 +109,72 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T .backend() suite("GetPartitioningCheckpointsEndpointSuite")( + test("Returns an expected PaginatedResponse[CheckpointV2DTO] with checkpoint-properties having multiple values") { + val properties = Map("executionID" -> Seq("id1", "id2")) + val encodedProps = properties.asBase64EncodedJsonString + val request = basicRequest + .get(uri"https://test.com/api/v2/partitionings/1/checkpoints?limit=10&offset=0&checkpoint-properties=$encodedProps") + .response(asJson[PaginatedResponse[CheckpointV2DTO]]) + + val response = request + .send(backendStub) + + val body = response.map(_.body) + val statusCode = response.map(_.code) + + assertZIO(body <&> statusCode)( + equalTo( + Right(PaginatedResponse(Seq(checkpointV2DTO1), Pagination(10, 0, hasMore = true), uuid)), + StatusCode.Ok + ) + ) + }, + test("Returns an expected PaginatedResponse[CheckpointV2DTO] when filtering by a process start time window") { + val request = basicRequest + .get( + uri"https://test.com/api/v2/partitionings/1/checkpoints?limit=10&offset=0&from=2026-06-01T00:00:00Z&to=2026-08-01T00:00:00Z" + ) + .response(asJson[PaginatedResponse[CheckpointV2DTO]]) + + val response = request + .send(backendStub) + + val body = response.map(_.body) + val statusCode = response.map(_.code) + + assertZIO(body <&> statusCode)( + equalTo( + Right(PaginatedResponse(Seq(checkpointV2DTO2), Pagination(10, 0, hasMore = false), uuid)), + StatusCode.Ok + ) + ) + }, + test("Returns expected 400 when 'from' is not a valid date-time") { + val request = basicRequest + .get(uri"https://test.com/api/v2/partitionings/1/checkpoints?limit=10&offset=0&from=2026-06-01") + .response(asJson[PaginatedResponse[CheckpointV2DTO]]) + + val response = request + .send(backendStub) + + val statusCode = response.map(_.code) + + assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) + }, + test("Returns expected 400 when 'from' is not before 'to'") { + val request = basicRequest + .get( + uri"https://test.com/api/v2/partitionings/1/checkpoints?limit=10&offset=0&from=2026-08-01T00:00:00Z&to=2026-08-01T00:00:00Z" + ) + .response(asJson[PaginatedResponse[CheckpointV2DTO]]) + + val response = request + .send(backendStub) + + val statusCode = response.map(_.code) + + assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) + }, test("Returns an expected PaginatedResponse[CheckpointV2DTO] with more data available") { val request = basicRequest .get(uri"https://test.com/api/v2/partitionings/1/checkpoints?limit=10&offset=0") @@ -187,6 +271,19 @@ object GetPartitioningCheckpointsEndpointUnitTests extends ZIOSpecDefault with T assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) }, + test("Returns expected 400 when a checkpoint property has no accepted value") { + val encodedProps = Map("executionID" -> Seq.empty[String]).asBase64EncodedJsonString + val request = basicRequest + .get(uri"https://test.com/api/v2/partitionings/1/checkpoints?limit=10&offset=0&checkpoint-properties=$encodedProps") + .response(asJson[PaginatedResponse[CheckpointV2DTO]]) + + val response = request + .send(backendStub) + + val statusCode = response.map(_.code) + + assertZIO(statusCode)(equalTo(StatusCode.BadRequest)) + }, test( "Returns an expected PaginatedResponse[CheckpointV2DTO] with more data available " + "(call without optional params)" diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryUnitTests.scala index af5308870..ef54fb94e 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/CheckpointRepositoryUnitTests.scala @@ -62,11 +62,11 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { when(getCheckpointMockV2.apply(GetPartitioningCheckpointV2Args(partitioningId, checkpointV2DTO3.id))) .thenReturn(ZIO.fail(new Exception("boom!"))) - when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(0L, 10, 0L, None, None, None))) + when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(0L, 10, 0L, None, None, None, None, None))) .thenReturn(ZIO.right(Seq(Row(FunctionStatus(11, "ok"), Some(checkpointItemPaginatedFromDB))))) - when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(1L, 10, 0L, None, None, None))) + when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(1L, 10, 0L, None, None, None, None, None))) .thenReturn(ZIO.right(Seq(Row(FunctionStatus(11, "ok"), None)))) - when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(3L, 10, 0L, None, None, None))) + when(getPartitioningCheckpointsMock.apply(GetPartitioningCheckpointsArgs(3L, 10, 0L, None, None, None, None, None))) .thenReturn(ZIO.left(DataNotFoundException(FunctionStatus(41, "Partitioning not found")))) when(getCheckpointPropertiesMock.apply(GetCheckpointPropertiesArgs(checkpointV2DTO1.id))) @@ -137,14 +137,14 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { suite("GetPartitioningCheckpointsSuite")( test("Returns expected Seq without properties") { for { - result <- CheckpointRepository.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, includeProperties = false) + result <- CheckpointRepository.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, None, None, includeProperties = false) } yield assertTrue( result.isInstanceOf[ResultHasMore[CheckpointV2DTO]] && result.data == Seq(checkpointV2DTO2) ) }, test("Returns expected Seq with properties") { for { - result <- CheckpointRepository.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, includeProperties = true) + result <- CheckpointRepository.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, None, None, includeProperties = true) } yield assertTrue( result.isInstanceOf[ResultHasMore[CheckpointV2DTO]] && result.data == Seq( checkpointV2DTO2.copy(properties = Some(Map("propName1" -> "propValue1"))) @@ -153,13 +153,13 @@ object CheckpointRepositoryUnitTests extends ZIOSpecDefault with TestData { }, test("Returns expected Seq.empty") { for { - result <- CheckpointRepository.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, includeProperties = false) + result <- CheckpointRepository.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, None, None, includeProperties = false) } yield assertTrue( result.isInstanceOf[ResultNoMore[CheckpointV2DTO]] && result.data == Seq.empty[CheckpointV2DTO] ) }, test("Returns expected NotFoundDatabaseError") { - assertZIO(CheckpointRepository.getPartitioningCheckpoints(3L, 10, 0L, None, None, None, includeProperties = false).exit)( + assertZIO(CheckpointRepository.getPartitioningCheckpoints(3L, 10, 0L, None, None, None, None, None, includeProperties = false).exit)( failsWithA[NotFoundDatabaseError] ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryUnitTests.scala index ff30f4b86..12c594385 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/repository/FlowRepositoryUnitTests.scala @@ -38,7 +38,7 @@ object FlowRepositoryUnitTests extends ZIOSpecDefault with TestData { private val getFlowCheckpointsV2Mock = mock(classOf[GetFlowCheckpoints]) private val getCheckpointPropertiesMock: GetCheckpointProperties = mock(classOf[GetCheckpointProperties]) - when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(1, 1, 1L, None, None))) + when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(1, 1, 1L, None, None, None, None, None))) .thenReturn( ZIO.right( Seq( @@ -46,9 +46,9 @@ object FlowRepositoryUnitTests extends ZIOSpecDefault with TestData { ) ) ) - when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(2, 1, 1L, None, None))) + when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(2, 1, 1L, None, None, None, None, None))) .thenReturn(ZIO.right(Seq(Row(FunctionStatus(11, "success"), None)))) - when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(3, 1, 1L, None, None))) + when(getFlowCheckpointsV2Mock.apply(GetFlowCheckpointsArgs(3, 1, 1L, None, None, None, None, None))) .thenReturn(ZIO.fail(DataNotFoundException(FunctionStatus(42, "Flow not found")))) when( @@ -67,12 +67,12 @@ object FlowRepositoryUnitTests extends ZIOSpecDefault with TestData { suite("GetFlowCheckpointsV2Suite")( test("Returns expected Right with CheckpointV2DTO without properties") { for { - result <- FlowRepository.getFlowCheckpoints(1, 1, 1L, None, None, includeProperties = false) + result <- FlowRepository.getFlowCheckpoints(1, 1, 1L, None, None, None, None, None, includeProperties = false) } yield assertTrue(result == ResultHasMore(Seq(checkpointWithPartitioningDTO1))) }, test("Returns expected Right with CheckpointV2DTO with properties") { for { - result <- FlowRepository.getFlowCheckpoints(1, 1, 1L, None, None, includeProperties = true) + result <- FlowRepository.getFlowCheckpoints(1, 1, 1L, None, None, None, None, None, includeProperties = true) } yield assertTrue( result == ResultHasMore( Seq(checkpointWithPartitioningDTO1.copy(properties = Some(Map("propName1" -> "propValue1")))) @@ -81,11 +81,11 @@ object FlowRepositoryUnitTests extends ZIOSpecDefault with TestData { }, test("Returns expected Right with CheckpointV2DTO") { for { - result <- FlowRepository.getFlowCheckpoints(2, 1, 1L, None, None, includeProperties = false) + result <- FlowRepository.getFlowCheckpoints(2, 1, 1L, None, None, None, None, None, includeProperties = false) } yield assertTrue(result == ResultNoMore(Seq.empty[CheckpointWithPartitioningDTO])) }, test("Returns expected DatabaseError") { - assertZIO(FlowRepository.getFlowCheckpoints(3, 1, 1L, None, None, includeProperties = false).exit)( + assertZIO(FlowRepository.getFlowCheckpoints(3, 1, 1L, None, None, None, None, None, includeProperties = false).exit)( failsWithA[NotFoundDatabaseError] ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceUnitTests.scala index e2c5406dc..eaba19979 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/service/CheckpointServiceUnitTests.scala @@ -45,9 +45,9 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { when(checkpointRepositoryMock.getCheckpoint(partitioningId, checkpointV2DTO2.id, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundDatabaseError("not found"))) - when(checkpointRepositoryMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, includeProperties = false)) + when(checkpointRepositoryMock.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultHasMore(Seq(checkpointV2DTO1)))) - when(checkpointRepositoryMock.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, includeProperties = false)) + when(checkpointRepositoryMock.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundDatabaseError("Partitioning not found"))) private val checkpointRepositoryMockLayer = ZLayer.succeed(checkpointRepositoryMock) @@ -92,13 +92,13 @@ object CheckpointServiceUnitTests extends ZIOSpecDefault with TestData { suite("GetPartitioningCheckpointsSuite")( test("Returns expected Right with Seq[CheckpointDTO]") { for { - result <- CheckpointService.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, includeProperties = false) + result <- CheckpointService.getPartitioningCheckpoints(1L, 10, 0L, None, None, None, None, None, includeProperties = false) } yield assertTrue { result == ResultHasMore(Seq(checkpointV2DTO1)) } }, test("Returns expected NotFoundServiceError") { - assertZIO(CheckpointService.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, includeProperties = false).exit)( + assertZIO(CheckpointService.getPartitioningCheckpoints(0L, 10, 0L, None, None, None, None, None, includeProperties = false).exit)( failsWithA[NotFoundServiceError] ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/api/v2/service/FlowServiceUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/api/v2/service/FlowServiceUnitTests.scala index bcd867305..51233afbe 100644 --- a/server/src/test/scala/za/co/absa/atum/server/api/v2/service/FlowServiceUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/api/v2/service/FlowServiceUnitTests.scala @@ -29,9 +29,9 @@ import zio.test._ object FlowServiceUnitTests extends ZIOSpecDefault with TestData { private val flowRepositoryMock = mock(classOf[FlowRepository]) - when(flowRepositoryMock.getFlowCheckpoints(1L, 1, 1L, None, None, includeProperties = false)) + when(flowRepositoryMock.getFlowCheckpoints(1L, 1, 1L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.succeed(ResultHasMore(Seq(checkpointWithPartitioningDTO1)))) - when(flowRepositoryMock.getFlowCheckpoints(2L, 1, 1L, None, None, includeProperties = false)) + when(flowRepositoryMock.getFlowCheckpoints(2L, 1, 1L, None, None, None, None, None, includeProperties = false)) .thenReturn(ZIO.fail(NotFoundDatabaseError("Flow not found"))) private val flowRepositoryMockLayer = ZLayer.succeed(flowRepositoryMock) @@ -42,13 +42,13 @@ object FlowServiceUnitTests extends ZIOSpecDefault with TestData { suite("GetFlowCheckpointsV2Suite")( test("Returns expected PaginatedResult[CheckpointV2DTO]") { for { - result <- FlowService.getFlowCheckpoints(1L, 1, 1L, None, None, includeProperties = false) + result <- FlowService.getFlowCheckpoints(1L, 1, 1L, None, None, None, None, None, includeProperties = false) } yield assertTrue { result == ResultHasMore(Seq(checkpointWithPartitioningDTO1)) } }, test("Returns expected ServiceError") { - assertZIO(FlowService.getFlowCheckpoints(2L, 1, 1L, None, None, includeProperties = false).exit)( + assertZIO(FlowService.getFlowCheckpoints(2L, 1, 1L, None, None, None, None, None, includeProperties = false).exit)( failsWithA[NotFoundServiceError] ) } diff --git a/server/src/test/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDBUnitTests.scala b/server/src/test/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDBUnitTests.scala index 87d2b75c6..1be1b3625 100644 --- a/server/src/test/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDBUnitTests.scala +++ b/server/src/test/scala/za/co/absa/atum/server/model/database/CheckpointItemWithPartitioningFromDBUnitTests.scala @@ -24,7 +24,7 @@ import java.util.UUID class CheckpointItemWithPartitioningFromDBUnitTests extends AnyFunSuiteLike with TestData { - test("groupAndConvertItemsToCheckpointWithPartitioningDTOs should group by idCheckpoint and sort by checkpointStartTime in descending order") { + test("groupAndConvertItemsToCheckpointWithPartitioningDTOs should group by idCheckpoint and keep the order of the DB rows") { val checkpointItem1 = CheckpointItemWithPartitioningFromDB( idCheckpoint = UUID.randomUUID(), checkpointName = "checkpoint1", @@ -57,14 +57,20 @@ class CheckpointItemWithPartitioningFromDBUnitTests extends AnyFunSuiteLike with hasMore = true ) - val checkpointItems = Seq(checkpointItem1, checkpointItem2) + // a second measure of the first checkpoint, i.e. another DB row of the same checkpoint + val checkpointItem1OtherMeasure = checkpointItem1.copy(measureName = "measure3", measuredColumns = Seq("col3")) - val result = CheckpointItemWithPartitioningFromDB.groupAndConvertItemsToCheckpointWithPartitioningDTOs(checkpointItems) + // earliest first, as returned by the DB for latest-first = false + val earliestFirst = CheckpointItemWithPartitioningFromDB + .groupAndConvertItemsToCheckpointWithPartitioningDTOs(Seq(checkpointItem1, checkpointItem1OtherMeasure, checkpointItem2)) + assert(earliestFirst.isRight) + assert(earliestFirst.toOption.get.map(_.id) == Seq(checkpointItem1.idCheckpoint, checkpointItem2.idCheckpoint)) + assert(earliestFirst.toOption.get.head.measurements.size == 2) - assert(result.isRight) - val checkpoints = result.toOption.get - - assert(checkpoints.size == 2) - assert(checkpoints.head.processStartTime.isAfter(checkpoints(1).processStartTime)) + // latest first, as returned by the DB by default + val latestFirst = CheckpointItemWithPartitioningFromDB + .groupAndConvertItemsToCheckpointWithPartitioningDTOs(Seq(checkpointItem2, checkpointItem1, checkpointItem1OtherMeasure)) + assert(latestFirst.isRight) + assert(latestFirst.toOption.get.map(_.id) == Seq(checkpointItem2.idCheckpoint, checkpointItem1.idCheckpoint)) } }