diff --git a/api/actions.py b/api/actions.py index d8a715b54..b8be83a60 100644 --- a/api/actions.py +++ b/api/actions.py @@ -1522,7 +1522,6 @@ def set_table_metadata(table: str, metadata): Args: table(str): name of table metadata: OEPMetadata or metadata object (dict) or metadata str - cursor: sql alchemy connection cursor """ # --------------------------------------- diff --git a/api/serializers.py b/api/serializers.py index 54a26b030..5fb681526 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -9,7 +9,7 @@ from django.urls import reverse from rest_framework import serializers -from dataedit.models import Table +from dataedit.models import Dataset, Table from modelview.models import Energyframework, Energymodel from oeplatform.settings import URL @@ -164,3 +164,36 @@ def validate_dataset(self, value): raise serializers.ValidationError("Dataset names must be unique.") return value + + +class DatasetReadSerializer(serializers.ModelSerializer): + class Meta: + model = Dataset + fields = ["uuid", "name", "metadata", "created_at"] + + +class DatasetCreateSerializer(serializers.Serializer): + name = serializers.CharField() + title = serializers.CharField() + description = serializers.CharField() + at_id = serializers.URLField(required=False) + + +class DatasetAssignTablesSerializer(serializers.Serializer): + tables = serializers.ListField( + child=serializers.DictField(child=serializers.CharField()), min_length=1 + ) + + def validate_tables(self, value): + for item in value: + if "name" not in item: + raise serializers.ValidationError("Each table must have 'name'.") + return value + + +class DatasetResourceSerializer(serializers.ModelSerializer): + schema = serializers.StringRelatedField() + + class Meta: + model = Table + fields = ["id", "schema", "name", "oemetadata", "human_readable_name"] diff --git a/api/services/dataset_creation.py b/api/services/dataset_creation.py new file mode 100644 index 000000000..58e22f4e3 --- /dev/null +++ b/api/services/dataset_creation.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: 2025 Jonas Huber © Reiner Lemoine Institut # noqa: E501 +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +from copy import deepcopy +from typing import Any + +from oemetadata.v2.v20.example import OEMETADATA_V20_EXAMPLE +from oemetadata.v2.v20.template import OEMETADATA_V20_TEMPLATE + + +def assemble_dataset_metadata( + validated_data: dict[str, Any], oemetadata: dict = OEMETADATA_V20_TEMPLATE +) -> dict[str, Any]: + # set the context + oemetadata = deepcopy(oemetadata) + oemetadata["@context"] = OEMETADATA_V20_EXAMPLE["@context"] + oemetadata["resources"] = [] # Remove resources + + oemetadata["@id"] = validated_data.get("at_id") + oemetadata["name"] = validated_data["name"] + oemetadata["title"] = validated_data["title"] + oemetadata["description"] = validated_data["description"] + + return oemetadata diff --git a/api/tests/test_datasets_api.py b/api/tests/test_datasets_api.py new file mode 100644 index 000000000..96ebef85d --- /dev/null +++ b/api/tests/test_datasets_api.py @@ -0,0 +1,173 @@ +# SPDX-FileCopyrightText: 2025 Jonas Huber © Reiner Lemoine Institut # noqa: E501 +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +from copy import deepcopy + +from oemetadata.latest.template import OEMETADATA_LATEST_TEMPLATE +from rest_framework import status +from rest_framework.test import APITestCase + +from dataedit.models import Dataset, Table, Topic + + +class DatasetAPITests(APITestCase): + def setUpDatasetMetadata(self, dataset_name: str): + metadata = deepcopy(OEMETADATA_LATEST_TEMPLATE) + + metadata["name"] = dataset_name + metadata["resources"] = [] + + return metadata + + def setUpResourceMetadata(self, table_name: str): + metadata = deepcopy(OEMETADATA_LATEST_TEMPLATE) + + metadata["resources"][0]["name"] = table_name + + return metadata + + def test_create_dataset(self): + payload = { + "name": "test_dataset", + "title": "Test Dataset", + "description": "This is a test dataset", + } + response = self.client.post( + "/api/v0/datasets/", payload, format="json" + ) # fixed + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertIn("metadata", response.data) + self.assertIn("resources", response.data["metadata"]) + self.assertEqual(response.data["metadata"]["name"], "test_dataset") + + def test_list_datasets(self): + Dataset.objects.create(name="ds1", metadata=self.setUpDatasetMetadata("ds1")) + Dataset.objects.create(name="ds2", metadata=self.setUpDatasetMetadata("ds2")) + response = self.client.get("/api/v0/datasets/") # fixed + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data), 2) + + def test_assign_tables_to_dataset(self): + # schema = Topic.objects.create(name="test_schema") + Table.objects.create(name="t1", oemetadata=self.setUpResourceMetadata("t1")) + Table.objects.create(name="t2", oemetadata=self.setUpResourceMetadata("t2")) + dataset = Dataset.objects.create( + name="test_dataset", metadata={"name": "test_dataset"} + ) + + payload = { + "dataset_name": "test_dataset", + "tables": [ + {"name": "t1"}, + {"name": "t2"}, + ], + } + + response = self.client.post( + "/api/v0/datasets/test_dataset/assign-tables/", payload, format="json" + ) + self.assertEqual(response.status_code, 200) + dataset.refresh_from_db() + self.assertEqual(len(dataset.tables.all()), 2) + self.assertEqual(len(dataset.metadata["resources"]), 2) + + def test_list_resources_for_dataset(self): + schema = Topic.objects.create(name="test_schema") + table = Table.objects.create( + name="t1", oemetadata=self.setUpResourceMetadata("t1") + ) + table.topics.add(schema) + dataset = Dataset.objects.create( + name="test_dataset", metadata=self.setUpDatasetMetadata("test_dataset") + ) + dataset.tables.add(table) + dataset.update_resources_from_tables() + + response = self.client.get( + f"/api/v0/datasets/{dataset.name}/resources/" + ) # fixed + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 1) + self.assertEqual(response.data[0]["name"], "t1") + + def test_assign_missing_table(self): + Dataset.objects.create( + name="ds_missing", metadata=self.setUpDatasetMetadata("ds_missing") + ) + + payload = { + "dataset_name": "ds_missing", + "tables": [{"name": "missing"}], + } + + response = self.client.post( + "/api/v0/datasets/ds_missing/assign-tables/", payload, format="json" + ) + self.assertEqual(response.status_code, 200) + self.assertIn("missing", response.data) + self.assertEqual(len(response.data["missing"]), 1) + + def test_list_resources_dataset_not_found(self): + response = self.client.get("/api/v0/datasets/nonexistent/resources/") # fixed + self.assertEqual(response.status_code, 404) + + +class DatasetManagerAPITests(APITestCase): + def setUp(self): + self.dataset = Dataset.objects.create( + name="test_dataset", + metadata={ + "name": "test_dataset", + "title": "Test Title", + "description": "Test Description", + "resources": [], + }, + ) + self.detail_url = f"/api/v0/datasets/{self.dataset.name}/" + + def test_get_dataset(self): + response = self.client.get(self.detail_url) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["name"], "test_dataset") + + def test_update_dataset(self): + updated_data = { + "name": "test_dataset", # must match existing name + "title": "Updated Title", + "description": "Updated Description", + "at_id": "https://example.org/dataset/test_dataset", + } + + response = self.client.put(self.detail_url, updated_data, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.dataset.refresh_from_db() + self.assertEqual(self.dataset.metadata["title"], "Updated Title") + self.assertEqual(self.dataset.metadata["description"], "Updated Description") + self.assertEqual( + self.dataset.metadata["@id"], "https://example.org/dataset/test_dataset" + ) + + def test_delete_dataset(self): + response = self.client.delete(self.detail_url) + self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) + self.assertFalse(Dataset.objects.filter(name="test_dataset").exists()) + + def test_get_nonexistent_dataset(self): + response = self.client.get("/api/v0/datasets/nonexistent_dataset/") + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + def test_put_nonexistent_dataset(self): + payload = { + "name": "nonexistent_dataset", + "title": "Does Not Exist", + "description": "Should return 404", + } + response = self.client.put( + "/api/v0/datasets/nonexistent_dataset/", payload, format="json" + ) + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + def test_delete_nonexistent_dataset(self): + response = self.client.delete("/api/v0/datasets/nonexistent_dataset/") + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) diff --git a/api/urls.py b/api/urls.py index 3d3a74d11..b7a9a74c2 100644 --- a/api/urls.py +++ b/api/urls.py @@ -49,6 +49,10 @@ AdvancedSetIsolationLevelAPIView, AdvancedUpdateAPIView, AllTableSizesAPIView, + AssignDatasetTables, + DatasetManager, + DatasetsListCreate, + DatasetsListResources, EnergyframeworkFactsheetListAPIView, EnergymodelFactsheetListAPIView, ManageOekgScenarioDatasetsAPIView, @@ -287,6 +291,26 @@ ManageOekgScenarioDatasetsAPIView.as_view(), name="add-scenario-datasets", ), + path( + "datasets/", + DatasetsListCreate.as_view(), + name="dataset-list-create", + ), + path( + "datasets//assign-tables/", + AssignDatasetTables.as_view(), + name="dataset-assign-tables", + ), + path( + "datasets//", + DatasetManager.as_view(), + name="dataset", + ), + path( + "datasets//resources/", + DatasetsListResources.as_view(), + name="dataset-resources", + ), path("db/table-sizes/", AllTableSizesAPIView.as_view(), name="table-sizes"), ] diff --git a/api/views.py b/api/views.py index 449a328e8..9c70715c5 100644 --- a/api/views.py +++ b/api/views.py @@ -39,6 +39,7 @@ import itertools import json import re +from copy import deepcopy import geoalchemy2 # noqa:F401 Although this import seems unused is has to be here import requests @@ -47,8 +48,10 @@ from django.contrib.postgres.search import TrigramSimilarity from django.db.models import Q from django.http import Http404, HttpRequest, JsonResponse +from django.shortcuts import get_object_or_404 from django.utils.decorators import method_decorator from django.views.decorators.cache import never_cache +from oemetadata.latest.example import OEMETADATA_LATEST_EXAMPLE from oemetadata.latest.template import OEMETADATA_LATEST_TEMPLATE from rest_framework import generics, status from rest_framework.authentication import TokenAuthentication @@ -145,11 +148,16 @@ query_typecast_select, ) from api.serializers import ( + DatasetAssignTablesSerializer, + DatasetCreateSerializer, + DatasetReadSerializer, + DatasetResourceSerializer, EnergyframeworkSerializer, EnergymodelSerializer, ScenarioBundleScenarioDatasetSerializer, ScenarioDataTablesSerializer, ) +from api.services.dataset_creation import assemble_dataset_metadata from api.services.embargo import ( EmbargoValidationError, apply_embargo, @@ -163,8 +171,10 @@ table_or_404, ) from api.validators.column import validate_column_names -from api.validators.identifier import assert_valid_table_name -from dataedit.models import Table +from api.validators.identifier import ( + assert_valid_table_name, +) +from dataedit.models import Dataset, Table from factsheet.permission_decorator import post_only_if_user_is_owner_of_scenario_bundle from modelview.models import Energyframework, Energymodel from oekg.utils import ( @@ -188,6 +198,14 @@ class TableMetadataAPIView(APIView): + """ + Important note: + oemetadata v2 introduces datasets which are not relevant on a table level + always query for metadata["resources"][0]. Keeping the complete oemetadata v2 JSON + makes it easy to integrate as no further changes to validation are required for now. + Datasets are handled in the model.Datasets & api views. + """ + @api_exception @method_decorator(never_cache) def get(self, request: Request, table: str) -> JsonLikeResponse: @@ -215,6 +233,9 @@ def post(self, request: Request, table: str) -> JsonLikeResponse: if metadata is not None: # update/sync keywords with tags before saving metadata + # oemetadata v2 introduces datasets which are not relevant on a table level + # always query for metadata["resources"][0] + keywords = metadata["resources"][0].get("keywords", []) or [] metadata["resources"][0]["keywords"] = update_tags_from_keywords( table=table_obj.name, keywords=keywords @@ -233,6 +254,99 @@ def post(self, request: Request, table: str) -> JsonLikeResponse: raise APIError(error) +class DatasetsListCreate(generics.ListCreateAPIView): + queryset = Dataset.objects.all() + + def get_serializer_class(self): + if self.request.method == "POST": + return DatasetCreateSerializer + return DatasetReadSerializer + + def create(self, request, *args, **kwargs): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + metadata = assemble_dataset_metadata(serializer.validated_data) + dataset = Dataset.objects.create(metadata=metadata, name=metadata["name"]) + + return Response( + {"id": dataset.pk, "metadata": dataset.metadata}, + status=status.HTTP_201_CREATED, + ) + + +class DatasetsListResources(generics.ListAPIView): + serializer_class = DatasetResourceSerializer + + def get_queryset(self): + dataset_name = self.kwargs["dataset_name"] + dataset = get_object_or_404(Dataset, name=dataset_name) + return dataset.tables.all() + + +class DatasetManager(APIView): + """ + View to retrieve, update, or delete a single dataset's metadata. + URL: /v0/datasets// + """ + + def get(self, request, dataset_name): + dataset = get_object_or_404(Dataset, name=dataset_name) + serializer = DatasetReadSerializer(dataset) + return Response(serializer.data, status=status.HTTP_200_OK) + + def put(self, request, dataset_name): + dataset = get_object_or_404(Dataset, name=dataset_name) + serializer = DatasetCreateSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + + dataset.metadata = assemble_dataset_metadata(serializer.validated_data) + dataset.save() + return Response({"message": "Dataset updated"}, status=status.HTTP_200_OK) + + def delete(self, request, dataset_name): + dataset = get_object_or_404(Dataset, name=dataset_name) + dataset.delete() + return Response( + {"message": "Dataset deleted"}, status=status.HTTP_204_NO_CONTENT + ) + + +class AssignDatasetTables(APIView): + def post(self, request, dataset_name): + serializer = DatasetAssignTablesSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + + table_refs = serializer.validated_data["tables"] + + try: + dataset = Dataset.objects.get(name=dataset_name) + except Dataset.DoesNotExist: + return Response({"error": "Dataset not found"}, status=404) + + missing = [] + added_tables = [] + + for table_ref in table_refs: + try: + table = Table.load(table_ref["name"]) + dataset.tables.add(table) + added_tables.append(table.name) + except Table.DoesNotExist: + missing.append(table_ref) + + dataset.update_resources_from_tables() + + return Response( + { + "message": f"Added {len(added_tables)} tables.", + "added": added_tables, + "missing": missing, + }, + status=200, + ) + + class TableAPIView(APIView): """ Handles the creation of tables and serves information on existing tables @@ -387,6 +501,36 @@ def put(self, request: Request, table: str) -> JsonLikeResponse: metadata = payload_query.get("metadata") if metadata: set_table_metadata(table=table, metadata=metadata) + else: + # If no metadata is provided, we create a minimal metadata object + metadata = deepcopy(OEMETADATA_LATEST_TEMPLATE) + metadata["@context"] = OEMETADATA_LATEST_EXAMPLE["@context"] + metadata["metaMetadata"] = OEMETADATA_LATEST_EXAMPLE["metaMetadata"] + + # Set basic resource info + resource = { + "name": table, + } + + # Update the first resource - there will only be one resource. + # The dataset section is managed by the database implementation ... + metadata["resources"][0].update(resource) + + # Build schema fields from columns + fields = [] + for col in columns: + field = { + "name": col["name"], + "type": col["data_type"], + "nullable": col.get("is_nullable", True), + # add more field metadata as needed + } + fields.append(field) + + # Replace the fields list entirely + metadata["resources"][0]["schema"]["fields"] = fields + + set_table_metadata(table=table, metadata=metadata) return JsonResponse({}, status=status.HTTP_201_CREATED) diff --git a/dataedit/migrations/0037_dataset.py b/dataedit/migrations/0037_dataset.py new file mode 100644 index 000000000..be2b20cf6 --- /dev/null +++ b/dataedit/migrations/0037_dataset.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: 2025 Jonas Huber © Reiner Lemoine Institut # noqa: E501 +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# Generated by Django 5.1.4 on 2025-07-31 12:27 + +import uuid + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("dataedit", "0036_alter_peerreview_oemetadata"), + ] + + operations = [ + migrations.CreateModel( + name="Dataset", + fields=[ + ( + "uuid", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("name", models.CharField(max_length=255, unique=True)), + ("metadata", models.JSONField(default=dict)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ( + "tables", + models.ManyToManyField( + blank=True, related_name="datasets", to="dataedit.table" + ), + ), + ], + ), + ] diff --git a/dataedit/migrations/0046_merge_0037_dataset_0045_alter_embargo_table.py b/dataedit/migrations/0046_merge_0037_dataset_0045_alter_embargo_table.py new file mode 100644 index 000000000..19819f93f --- /dev/null +++ b/dataedit/migrations/0046_merge_0037_dataset_0045_alter_embargo_table.py @@ -0,0 +1,18 @@ +""" +SPDX-FileCopyrightText: 2025 Jonas Huber © Reiner Lemoine Institut +SPDX-License-Identifier: AGPL-3.0-or-later +""" # noqa: 501 + +# Generated by Django 5.1.4 on 2026-06-10 15:18 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("dataedit", "0037_dataset"), + ("dataedit", "0045_alter_embargo_table"), + ] + + operations = [] diff --git a/dataedit/models.py b/dataedit/models.py index 119fd266c..8286963e1 100644 --- a/dataedit/models.py +++ b/dataedit/models.py @@ -19,6 +19,7 @@ import json import logging import re +import uuid from datetime import datetime, timedelta from enum import Enum from typing import TYPE_CHECKING, Literal, Mapping, Union @@ -456,6 +457,28 @@ def _get_oeb_table_proxy_w_permission( ) +class Dataset(models.Model): + """Represents a dataset in the database. + + Datasets are implemented according to oemetadata specification. + """ + + uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=255, unique=True) + metadata = models.JSONField(null=False, default=dict) + tables = models.ManyToManyField("Table", related_name="datasets", blank=True) + created_at = models.DateTimeField(auto_now_add=True) + + def update_resources_from_tables(self): + """ + Rebuild the `resources` field in OEMetadata based on linked tables. + """ + self.metadata["resources"] = [ + table.oemetadata["resources"][0] for table in self.tables.all() + ] + self.save() + + class Embargo(models.Model): DURATION_CHOICES = [ ("6_months", "6 Months"), diff --git a/versions/changelogs/current.md b/versions/changelogs/current.md index 9f9f27062..93bb6ad33 100644 --- a/versions/changelogs/current.md +++ b/versions/changelogs/current.md @@ -11,9 +11,78 @@ SPDX-License-Identifier: CC0-1.0 ## Changes -## Features +### Features -## Bugs +- Build search field to search in the user's tables + [(#2248)](https://github.com/OpenEnergyPlatform/oeplatform/pull/2248) + +- Update the Graph Vie WIdget on the OEO Viewer page and enable the graph + comparison feature + [(#2277)](https://github.com/OpenEnergyPlatform/oeplatform/pull/2277). + +- Add a Option to select the Language on the OEO Entity Pages. This only shows + the german / english synonym if available for an entity + [(#2277)](https://github.com/OpenEnergyPlatform/oeplatform/pull/2277). + +- Add Dataset rest-api and metadata based concept as specified in oemetadata / + frictionless + ([#2071](https://github.com/OpenEnergyPlatform/oeplatform/pull/2071)) + - Resource metadata is stored for each created table. + - Dataset objects can be listed, created, edited and existing tables can be + assigned as resource + - Datasets and assigned Resources are stored in the django database using a + m:n relation with tables to read the oemetadata. + - Rest api implementation +- The metadata api now also syncs the data schema documented in the metadata + with the table schema available in the database + [(#2290)](https://github.com/OpenEnergyPlatform/oeplatform/pull/2290) + +- Add new java script functionality for enhanced fetching of additional + information from table metadata and reworked table listing UI to show table + listing cards with additional information + [(#2311)](https://github.com/OpenEnergyPlatform/oeplatform/pull/2311) + +### Bugs + +- Reviewer&Contributor page: calculation of percentage of progress of reviewed + fields takes into account empty fields + [(#1386)](https://github.com/OpenEnergyPlatform/oeplatform/pull/1386) + +- Refactored the OEO Viewer layout to better organize hierarchy, metadata, and + graph widgets, including improved mobile responsiveness and also adapt to the + oeo inferred version which is now served by the TIB-TS (OLSv4 System) + [(#2237)](https://github.com/OpenEnergyPlatform/oeplatform/pull/2277). + +- Update the Hierarchy Widget to Expand and highlight the currently selected + Entity in the OEO Viewer + [(#2237)](https://github.com/OpenEnergyPlatform/oeplatform/pull/2277). + +- On the OEO Entity page the Entity type is now automatically detected to stream + line the user experience as users do not have to check the type manually - we + now also show the type of the entity + [(#2237)](https://github.com/OpenEnergyPlatform/oeplatform/pull/2277). + +- Fixed a bug in the TIB-TS api when the user navigates to ObjectProperties / + Individuals. The API path is now correctly set + [(#2237)](https://github.com/OpenEnergyPlatform/oeplatform/pull/2277). + +- Reviewer&Contributor page: calculation of percentage of progress of reviewed + fields takes into account empty fields + [(#1386)](https://github.com/OpenEnergyPlatform/oeplatform/pull/1386) + +- Cleanup incomplete updates to the OpenPeerReview. Some parts of the code are + incomplete due to a messi code refactoring where some code snippets have been + lost due to merge conflicts in commits that are not pushed to remote. + [(#2289)](https://github.com/OpenEnergyPlatform/oeplatform/pull/2289) + +- Fixed a bug in the oemetaBuilder tool that removed `isAbout` and + `valueReference` entries and added unwanted properties when the users submits + the Editor form + [(#2290)](https://github.com/OpenEnergyPlatform/oeplatform/pull/2290) + +- Fixed a bug in OEO loading module that was made visible by OEO version 2.12.0 + as there was a new metadata owl file introduced which can not be parsed with + rdflib [(#2311)](https://github.com/OpenEnergyPlatform/oeplatform/pull/2311) - Fix navigation box during the OPR; show proper information and jump to next field that needs review. Hide start button for OPR if metadata is empty.