diff --git a/python/grass/temporal/list_stds.py b/python/grass/temporal/list_stds.py index afe7e5abb55..54773e2d1c7 100644 --- a/python/grass/temporal/list_stds.py +++ b/python/grass/temporal/list_stds.py @@ -46,18 +46,23 @@ def get_dataset_list( This method returns a dictionary, the keys are the available mapsets, the values are the rows from the SQL database query. - :param type: The type of the datasets (strds, str3ds, stvds, raster, - raster_3d, vector) + :param type: The dataset type(s) (strds, str3ds, stvds, raster, + raster_3d, vector) as a string with a single dataset type + (e.g. "strds") or a list of strings with dataset types (e.g. + ["strds", "stvds"]) :param temporal_type: The temporal type of the datasets (absolute, - relative) + relative) as a string or a list of strings :param columns: A comma separated list of columns that will be selected :param where: A where statement for selected listing without "WHERE" :param order: A comma separated list of columns to order the datasets by category :param dbif: The database interface to be used - :return: A dictionary with the rows of the SQL query for each - available mapset + :return: A dictionary with mapsets as keys. When *type* is a string, + values are raw database row objects (preserving backward + compatibility). When *type* is a list, values are plain + dicts with an additional ``"type"`` key identifying the + dataset type of each record. .. code-block:: pycon @@ -97,35 +102,105 @@ def get_dataset_list( >>> check = sp.delete() """ + msgr = get_tgis_message_interface() + dbif, connection_state_changed = init_dbif(dbif) + is_list_input = isinstance(type, list) + stds_type = [type] if isinstance(type, str) else type + if isinstance(temporal_type, str): + temporal_type = [temporal_type] + result = {} - for mapset in dbif.tgis_mapsets: - if temporal_type == "absolute": - table = type + "_view_abs_time" - else: - table = type + "_view_rel_time" + for ttype in temporal_type: + mapset_for_schema = ( + list(dbif.tgis_mapsets.keys())[0] if dbif.tgis_mapsets else None + ) + if not mapset_for_schema: + continue + + type_schemas = [] + for dtype in stds_type: + table = ( + dtype + "_view_abs_time" + if ttype == "absolute" + else dtype + "_view_rel_time" + ) + dbif.execute(f"SELECT * FROM {table} WHERE 0=1", mapset=mapset_for_schema) + type_schemas.append( + [d[0] for d in dbif.connections[mapset_for_schema].cursor.description] + ) - if columns and columns.find("all") == -1: - sql = "SELECT " + str(columns) + " FROM " + table - else: - sql = "SELECT * FROM " + table + common_columns = [ + col + for col in type_schemas[0] + if all(col in schema for schema in type_schemas[1:]) + ] + valid_columns_set = set(common_columns) - if where: - sql += " WHERE " + where - sql += " AND mapset = '%s'" % (mapset) + if columns and columns.find("all") == -1: + requested_columns = [ + col.strip() for col in columns.split(",") if col != "type" + ] + for col in requested_columns: + if col not in valid_columns_set: + if connection_state_changed: + dbif.close() + if len(stds_type) == 1: + msgr.fatal( + _( + "Column '%s' is not available for the requested dataset type" + ) + % col + ) + else: + msgr.fatal( + _( + "Column '%s' is not available for the requested combination of dataset types" + ) + % col + ) + final_columns = requested_columns else: - sql += " WHERE mapset = '%s'" % (mapset) - - if order: - sql += " ORDER BY " + order - - dbif.execute(sql, mapset=mapset) - rows = dbif.fetchall(mapset=mapset) - - if rows: - result[mapset] = rows + final_columns = common_columns + + if not final_columns: + if connection_state_changed: + dbif.close() + msgr.fatal(_("No valid database columns were requested")) + + columns_to_query = ",".join(final_columns) + + for dtype in stds_type: + for mapset in dbif.tgis_mapsets: + if ttype == "absolute": + table = dtype + "_view_abs_time" + else: + table = dtype + "_view_rel_time" + + sql = f"SELECT {columns_to_query} FROM {table}" + + if where: + sql += " WHERE " + where + sql += " AND mapset = '%s'" % (mapset) + else: + sql += " WHERE mapset = '%s'" % (mapset) + + if order: + sql += " ORDER BY " + order + + dbif.execute(sql, mapset=mapset) + rows = dbif.fetchall(mapset=mapset) + + if rows: + if mapset not in result: + result[mapset] = [] + if is_list_input: + for row in rows: + result[mapset].append({**dict(row), "type": dtype}) + else: + result[mapset].extend(rows) if connection_state_changed: dbif.close() diff --git a/temporal/t.list/t.list.md b/temporal/t.list/t.list.md index 675bbceabe2..66ef585c62e 100644 --- a/temporal/t.list/t.list.md +++ b/temporal/t.list/t.list.md @@ -2,8 +2,9 @@ *t.list* lists any dataset that is registered in the temporal database. Datasets are raster, 3D raster and vector maps as well as their -corresponding space time datasets (STRDS, STR3DS and STVDS). The type of -the dataset can be specified using the *type* option, default is STRDS. +corresponding space time datasets (STRDS, STR3DS and STVDS). The type of the +dataset can be specified using the *type* option (default is STRDS). +Multiple comma-separated types can be provided, such as *type=strds,stvds*. By default all datasets with relative and absolute time are listed. However, the user has the ability to specify a single temporal type with the *temporaltype* option. The user can define the columns that should @@ -149,7 +150,47 @@ t.list type=raster format=json columns=id,start_time "start_time": "2012-12-01 00:00:00" } ] +``` + +To list multiple dataset types at once, specify a comma-separated list of +types (e.g., `type=strds,stvds`). + +**Note:** The `type` column is automatically included in the **csv** and +**json** output formats when multiple types are requested to clearly identify +the dataset type of each record. It can also be explicitly requested for +single types using `columns=id,type`. + +```sh +t.list type=strds,stvds columns=id format=json +``` +```json +[ + { + "id": "lst_daily@PERMANENT", + "type": "strds" + }, + { + "id": "mini_set@PERMANENT", + "type": "strds" + }, + { + "id": "nc_lst_daily@PERMANENT", + "type": "strds" + }, + { + "id": "precip_abs1@PERMANENT", + "type": "strds" + }, + { + "id": "prec_observer@PERMANENT", + "type": "stvds" + }, + { + "id": "schools_stds@PERMANENT", + "type": "stvds" + } +] ``` ## SEE ALSO diff --git a/temporal/t.list/t.list.py b/temporal/t.list/t.list.py index 49c2eb76853..fe7203c5ad3 100755 --- a/temporal/t.list/t.list.py +++ b/temporal/t.list/t.list.py @@ -34,6 +34,7 @@ # % description: Type of the space time dataset or map, default is strds # % guisection: Selection # % required: no +# % multiple: yes # % options: strds, str3ds, stvds, raster, raster_3d, vector # % answer: strds # %end @@ -64,7 +65,7 @@ # % guisection: Selection # % required: no # % multiple: yes -# % options: id,name,semantic_label,creator,mapset,number_of_maps,creation_time,start_time,end_time,north,south,west,east,granularity,all +# % options: id,name,type,semantic_type,semantic_label,creator,mapset,number_of_maps,creation_time,start_time,end_time,north,south,west,east,granularity,all # % answer: # %end @@ -112,7 +113,7 @@ def main(): # Get the options - stds_type = options["type"] + stds_type = options["type"].split(",") temporal_type = options["temporaltype"] columns = options["columns"] order = options["order"] @@ -143,19 +144,41 @@ def main(): elif output_format == "line": if colhead: gs.fatal(_("Column names are not allowed with line format")) + if len(stds_type) > 1: + gs.fatal(_("Only one type is allowed for line format")) if not separator: separator = "," columns_list = columns.split(",") if columns else [] - if len(columns_list) > 1: - gs.fatal( - _( - "Only one column is allowed for line format (not {num_columns})" - ).format(num_columns=len(columns_list)) - ) + if columns == "all" or len(columns_list) > 1: + gs.fatal(_("Only one column is allowed for line format")) elif not separator: separator = "|" + if set(stds_type) & {"raster", "raster_3d", "vector"} and set(stds_type) & { + "strds", + "str3ds", + "stvds", + }: + gs.fatal( + _( + "Combinations across space time datasets and time stamped maps " + "(e.g., raster and strds) are not allowed" + ) + ) + if columns and "type" in columns.split(","): + cols_list = [c.strip() for c in columns.split(",") if c.strip() != "type"] + if not cols_list: + gs.fatal(_("Column 'type' cannot be requested alone")) + columns_for_db = ",".join(cols_list) + else: + columns_for_db = columns + + # If only one type is requested and 'type' is not in columns, pass it as a string + # so get_dataset_list doesn't implicitly inject the 'type' column. + if len(stds_type) == 1 and not (columns and "type" in columns.split(",")): + stds_type = stds_type[0] + # Lazy import and initialize TGIS import grass.temporal as tgis @@ -186,86 +209,102 @@ def main(): line_output = [] first = True - if gs.verbosity() > 0 and not outpath and output_format == "plain": - sys.stderr.write("----------------------------------------------\n") - - # Replace separate "if outpath" and "else" blocks with a unified context manager: with ( - open(outpath, "w") - if (outpath and outpath != "-") - else nullcontext(sys.stdout) as out_file - ): + open(outpath, "w") if (outpath and outpath != "-") else nullcontext(sys.stdout) + ) as out_file: for ttype in temporal_type.split(","): time = "absolute time" if ttype == "absolute" else "relative time" stds_list = tgis.get_dataset_list( - stds_type, ttype, columns, where, order, dbif=dbif + stds_type, ttype, columns_for_db, where, order, dbif=dbif ) for mapset in dbif.tgis_mapsets: rows = stds_list.get(mapset) if rows: - if ( - gs.verbosity() > 0 - and (not outpath or outpath == "-") - and output_format == "plain" - ): - if stds_type in {"raster", "raster_3d", "vector"}: - sys.stderr.write( - _( - "Time stamped %s maps with %s available in mapset " - "<%s>:\n" - ) - % (stds_type, time, mapset) - ) + if output_format == "plain": + if isinstance(stds_type, str): + groups = [(stds_type, rows)] else: + rows_by_type = {} + for r in rows: + rows_by_type.setdefault(r["type"], []).append(r) + groups = rows_by_type.items() + else: + # Process all rows together + groups = [ + ( + rows[0]["type"] + if not isinstance(stds_type, str) + else stds_type, + rows, + ) + ] + + for dtype, current_rows in groups: + if ( + gs.verbosity() > 0 + and (not outpath or outpath == "-") + and output_format == "plain" + ): sys.stderr.write( - _( - "Space time %s datasets with %s available in " - "mapset <%s>:\n" + "----------------------------------------------\n" + ) + if dtype in {"raster", "raster_3d", "vector"}: + sys.stderr.write( + _( + "Time stamped %s maps with %s available" + " in mapset <%s>:\n" + ) + % (dtype, time, mapset) ) - % ( - stds_type, - time, - mapset, + else: + type_desc = { + "strds": "raster", + "stvds": "vector", + "str3ds": "3D raster", + }.get(dtype, dtype) + sys.stderr.write( + _( + "Space time %s datasets with %s available" + " in mapset <%s>:\n" + ) + % (type_desc, time, mapset) ) - ) - if output_format == "json": - for row in rows: - json_output.append(dict(row)) + if output_format == "json": + json_output.extend([dict(row) for row in current_rows]) - elif output_format == "line": - line_output.extend([str(row[0]) for row in rows]) + elif output_format == "line": + line_output.extend( + str(v) + for row in current_rows + for v in dict(row).values() + ) - else: - if (colhead or output_format == "csv") and first: - output = "" - count = 0 - for col_key in rows[0].keys(): - output += (separator if count > 0 else "") + str( - col_key + else: + print_header = (output_format == "csv" and first) or ( + output_format == "plain" and colhead + ) + if print_header: + output = separator.join( + str(k) for k in current_rows[0].keys() ) - count += 1 - out_file.write(f"{output}\n") - first = False - - for row in rows: - output = "" - count = 0 - for col in row: - # If the database value is None, make it an empty string for csv - if col is None: - cell_value = ( - "" if output_format == "csv" else "None" - ) + out_file.write(f"{output}\n") + if output_format == "csv": + first = False else: - cell_value = str(col) - - output += (separator if count > 0 else "") + cell_value - count += 1 - out_file.write("{st}\n".format(st=output)) + colhead = False + + for row in current_rows: + output = separator.join( + ("" if output_format == "csv" else "None") + if v is None + else str(v) + for v in dict(row).values() + ) + out_file.write(f"{output}\n") - # Dump the collected JSON and line data + # Dump the collected output if output_format == "json": out_file.write(json.dumps(json_output, indent=4, default=str) + "\n") elif output_format == "line": diff --git a/temporal/t.list/tests/conftest.py b/temporal/t.list/tests/conftest.py index 04efd97ee8d..28e9da9112c 100644 --- a/temporal/t.list/tests/conftest.py +++ b/temporal/t.list/tests/conftest.py @@ -3,6 +3,7 @@ """ import os +from io import StringIO from types import SimpleNamespace import pytest @@ -13,11 +14,11 @@ @pytest.fixture(scope="module") def space_time_dataset(tmp_path_factory): - """Start an isolated session and create a raster time series. + """Start an isolated session and create raster and vector time series. - Returns an object with attributes about the dataset. + Returns an object with attributes about the datasets. """ - tmp_path = tmp_path_factory.mktemp("raster_time_series") + tmp_path = tmp_path_factory.mktemp("time_series") project = tmp_path / "test_project" gs.create_project(project) @@ -51,20 +52,47 @@ def space_time_dataset(tmp_path_factory): flags="i", ) + vector_names = [f"vect_{i}" for i in range(1, 3)] + coords = [f"{i * 10}|{i * 10}" for i in range(1, 3)] + for name, coord in zip(vector_names, coords, strict=True): + tools.v_in_ascii( + input=StringIO(coord), + output=name, + format="point", + ) + stvds_name = "vector_dataset" + tools.t_create( + type="stvds", + temporaltype="absolute", + output=stvds_name, + title="Vector dataset", + description="Vector series generated for tests", + ) + vector_file = tmp_path / "vector_names.txt" + vector_file.write_text("\n".join(vector_names)) + tools.t_register( + type="vector", + input=stvds_name, + file=vector_file, + start="2026-01-01", + increment="1 month", + flags="i", + ) + gs.run_command("g.mapset", mapset="user1", flags="c", env=session.env) - user_dataset_name = "precipitation_dataset" + raster_dataset_name = "precipitation_dataset" tools.r_mapcalc(expression="precip_1 = 2", overwrite=True) tools.t_create( type="strds", temporaltype="absolute", - output=user_dataset_name, + output=raster_dataset_name, title="Precip", description="user1 Dataset", ) tools.t_register( type="raster", - input=user_dataset_name, + input=raster_dataset_name, maps="precip_1", start="2026-01-01", increment="1 month", @@ -76,8 +104,12 @@ def space_time_dataset(tmp_path_factory): yield SimpleNamespace( session=session, name=dataset_name, + raster_dataset1=dataset_name, + raster_dataset2=raster_dataset_name, map_names=names, - user_dataset=user_dataset_name, + raster_dataset=raster_dataset_name, + stvds_name=stvds_name, + vector_names=vector_names, ) diff --git a/temporal/t.list/tests/test_t_list.py b/temporal/t.list/tests/test_t_list.py index 504c2efac92..05ae4c0d726 100644 --- a/temporal/t.list/tests/test_t_list.py +++ b/temporal/t.list/tests/test_t_list.py @@ -1,11 +1,9 @@ """Test t.list functionality""" -import json - import pytest from grass.experimental import TemporaryMapsetSession -from grass.tools import Tools +from grass.tools import ToolError, Tools def test_t_list_defaults(space_time_dataset): @@ -15,13 +13,13 @@ def test_t_list_defaults(space_time_dataset): strds_result = tools.t_list(type="strds", columns="name") assert strds_result.returncode == 0 - strds_lines = [line.strip() for line in strds_result.stdout.strip().splitlines()] - assert space_time_dataset.name in strds_lines + strds_lines = strds_result.text_split("\n") + assert space_time_dataset.raster_dataset1 in strds_lines # Test Map Listing (raster) raster_result = tools.t_list(type="raster", columns="name") - raster_lines = [line.strip() for line in raster_result.stdout.strip().splitlines()] + raster_lines = raster_result.text_split("\n") for map_name in space_time_dataset.map_names: assert map_name in raster_lines @@ -31,9 +29,9 @@ def test_t_list_where_filter(space_time_dataset): tools = Tools(session=space_time_dataset.session) match = tools.t_list(type="strds", columns="name", where="name LIKE 'temp_%'") - match_lines = [line.strip() for line in match.stdout.strip().splitlines()] + match_lines = match.text_split("\n") assert len(match_lines) == 1 - assert match_lines[0] == space_time_dataset.name + assert match_lines[0] == space_time_dataset.raster_dataset1 empty = tools.t_list(type="strds", columns="name", where="name LIKE 'land_%'") assert empty is None @@ -46,7 +44,7 @@ def test_t_list_order(space_time_dataset): result_asc = tools.t_list(type="raster", columns="name", order="start_time") assert result_asc is not None - lines_asc = [line.strip() for line in result_asc.stdout.strip().splitlines()] + lines_asc = result_asc.text_split("\n") assert lines_asc == space_time_dataset.map_names @@ -56,9 +54,7 @@ def test_t_list_json(space_time_dataset): tools = Tools(session=space_time_dataset.session) result = tools.t_list( type="raster", format="json", columns="name,start_time,end_time" - ) - - data = json.loads(result.stdout) + ).json expected = [ { @@ -78,10 +74,10 @@ def test_t_list_json(space_time_dataset): }, ] - assert len(data) == len(expected) + assert len(result) == len(expected) for i, expected_item in enumerate(expected): - actual_item = data[i] + actual_item = result[i] for key, expected_value in expected_item.items(): assert actual_item[key] == expected_value @@ -102,7 +98,7 @@ def test_t_list_csv(space_time_dataset, separator): ] expected = "\n".join(expected_lines) - assert result.stdout.strip() == expected + assert result.text == expected @pytest.mark.parametrize("separator", [",", ":"]) @@ -115,7 +111,7 @@ def test_t_list_line(space_time_dataset, separator): expected = separator.join(space_time_dataset.map_names) - assert result.stdout.strip() == expected + assert result.text == expected @pytest.mark.parametrize("output_format", ["json", "line", "plain"]) @@ -141,7 +137,7 @@ def test_t_list_mapset_current(space_time_dataset): result = tools.t_list(type="strds", mapset=".", columns="name", format="json") assert len(result) == 1 - assert result[0]["name"] == space_time_dataset.name + assert result[0]["name"] == space_time_dataset.raster_dataset1 def test_t_list_mapset_all(space_time_dataset): @@ -152,8 +148,8 @@ def test_t_list_mapset_all(space_time_dataset): assert len(result) == 2 names = [d["name"] for d in result] - assert space_time_dataset.name in names - assert space_time_dataset.user_dataset in names + assert space_time_dataset.raster_dataset1 in names + assert space_time_dataset.raster_dataset2 in names def test_t_list_mapset_explicit(space_time_dataset): @@ -162,13 +158,13 @@ def test_t_list_mapset_explicit(space_time_dataset): res_user = tools.t_list(type="strds", mapset="user1", columns="name", format="json") assert len(res_user) == 1 - assert res_user[0]["name"] == space_time_dataset.user_dataset + assert res_user[0]["name"] == space_time_dataset.raster_dataset2 res = tools.t_list(type="strds", mapset="PERMANENT,user1", format="json") assert len(res) == 2 names = [d["name"] for d in res] - assert space_time_dataset.name in names - assert space_time_dataset.user_dataset in names + assert space_time_dataset.raster_dataset1 in names + assert space_time_dataset.raster_dataset2 in names def test_t_list_empty_database(empty_session): @@ -193,9 +189,64 @@ def test_t_list_empty_database(empty_session): def test_t_list_from_mapset_without_temporal_database(space_time_dataset): """Datasets in accessible mapsets are listed even when the current mapset - has no temporal database (https://github.com/OSGeo/grass/issues/7622).""" + has no temporal database.""" with TemporaryMapsetSession(env=space_time_dataset.session.env) as mapset_session: tools = Tools(session=mapset_session) result = tools.t_list(type="strds", format="json") names = [record["name"] for record in result.json] - assert space_time_dataset.name in names + assert space_time_dataset.raster_dataset1 in names + + +def test_t_list_multiple_types_json(space_time_dataset): + """Check that listing multiple dataset types in JSON format returns entries from all types.""" + tools = Tools(session=space_time_dataset.session) + result = tools.t_list(type="stvds,strds", mapset=".", columns="name", format="json") + + assert len(result.json) == 2 + names = [record["name"] for record in result.json] + assert space_time_dataset.raster_dataset1 in names + assert space_time_dataset.stvds_name in names + + types = {record["type"] for record in result.json} + assert "strds" in types + assert "stvds" in types + + +def test_t_list_multiple_types_line_error(space_time_dataset): + """Check that line format with multiple dataset types raises an error.""" + tools = Tools(session=space_time_dataset.session) + with pytest.raises(ToolError, match="Only one type is allowed for line format"): + tools.t_list(type="strds,stvds", columns="id", format="line") + + +def test_t_list_line_columns_all_error(space_time_dataset): + """Check that line format with columns=all raises an error.""" + tools = Tools(session=space_time_dataset.session) + with pytest.raises(ToolError, match="Only one column is allowed for line format"): + tools.t_list(type="strds", format="line", columns="all") + + +def test_t_list_cross_category_types_error(space_time_dataset): + """Check that mixing space time datasets and time stamped maps raises an error.""" + tools = Tools(session=space_time_dataset.session) + with pytest.raises( + ToolError, + match="Combinations across space time datasets and time stamped maps", + ): + tools.t_list(type="strds,raster", columns="id") + + +def test_t_list_multiple_types_csv(space_time_dataset): + """Check that CSV with multiple types includes only shared columns.""" + tools = Tools(session=space_time_dataset.session) + result = tools.t_list(type="strds,stvds", mapset=".", format="csv", columns="all") + lines = result.text_split("\n") + header = lines[0].split(",") + + assert "id" in header + assert "name" in header + assert "type" in header + + # Header should NOT include dataset-specific columns + assert "raster_register" not in header + assert "vector_register" not in header