diff --git a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py index ecc9540bc0b7..2a8f13e1a79e 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py @@ -81,6 +81,7 @@ from metadata.ingestion.source.dashboard.powerbi.models import ( Dataflow, DataflowExportResponse, + Datamart, Dataset, Group, PowerBIDashboard, @@ -373,6 +374,12 @@ def _get_dataflow_url(self, workspace_id: str, dataflow_id: str) -> str: f"{workspace_id}/dataflows/{dataflow_id}?experience=power-bi" ) + def _get_datamart_url(self, workspace_id: str, datamart_id: str) -> str: + """ + Method to build the datamart url + """ + return f"{clean_uri(self.service_connection.hostPort)}/groups/{workspace_id}/datamarts/{datamart_id}" + def _get_chart_url(self, report_id: Optional[str], workspace_id: str, dashboard_id: str) -> str: # noqa: UP045 """ Method to build the chart url @@ -606,11 +613,15 @@ def _get_dataflow_column_info(self, dataflow_export: DataflowExportResponse) -> logger.warning(f"Error to yield dataflow entity column: {exc}") return datasource_columns - def _get_datamodels_list(self) -> List[Union[Dataset, Dataflow]]: # noqa: UP006, UP007 + def _get_datamodels_list(self) -> List[Union[Dataset, Dataflow, Datamart]]: # noqa: UP006, UP007 """ Get All the Powerbi Datasets """ - return self.context.get().workspace.datasets + self.context.get().workspace.dataflows + return ( + self.context.get().workspace.datasets + + self.context.get().workspace.dataflows + + (self.context.get().workspace.datamarts or []) + ) def yield_datamodel(self, dashboard_details: Group) -> Iterable[Either[CreateDashboardDataModelRequest]]: """ @@ -644,6 +655,13 @@ def yield_datamodel(self, dashboard_details: Group) -> Iterable[Either[CreateDas if dataflow_export: self.dataflow_exports[dataset.id] = dataflow_export datamodel_columns = self._get_dataflow_column_info(dataflow_export) + elif isinstance(dataset, Datamart): + data_model_type = DataModelType.PowerBIDatamart.value + datamodel_columns = [] + source_url = self._get_datamart_url( + workspace_id=self.context.get().workspace.id, + datamart_id=dataset.id, + ) else: logger.warning(f"Unknown dataset type: {type(dataset)}, name: {dataset.name}") continue @@ -1982,6 +2000,55 @@ def create_dataflow_upstream_dataflow_lineage( ) ) + def create_datamart_upstream_datamart_lineage( + self, datamodel: Datamart, datamodel_entity: DashboardDataModel + ) -> Iterable[Either[AddLineageRequest]]: + """ + Create lineage between datamart and upstreamDatamart + """ + for upstream_datamart in datamodel.upstreamDatamarts or []: + try: + if not upstream_datamart.targetDatamartId: + logger.debug( + f"No targetDatamartId found for upstreamDatamart in " + f"datamodel [{datamodel_entity.name.root}], " + f"Moving to next upstreamDatamart" + ) + continue + if upstream_datamart.targetDatamartId == datamodel.id: + continue + upstream_datamart_fqn = fqn.build( + self.metadata, + entity_type=DashboardDataModel, + service_name=self.context.get().dashboard_service, + data_model_name=upstream_datamart.targetDatamartId, + ) + upstream_datamart_entity = self.metadata.get_by_name( + entity=DashboardDataModel, + fqn=upstream_datamart_fqn, + ) + if upstream_datamart_entity and datamodel_entity: + yield self._get_add_lineage_request( + from_entity=upstream_datamart_entity, + to_entity=datamodel_entity, + ) + else: + logger.debug( + f"No upstreamDatamart entity with id={str(upstream_datamart.targetDatamartId)} " # noqa: RUF010 + f"found for datamodel [{datamodel_entity.name.root}]" + ) + except Exception as exc: # pylint: disable=broad-except + yield Either( + left=StackTraceError( + name="Datamart and UpstreamDatamart Lineage", + error=( + f"Error to yield datamart and upstreamDatamart lineage " + f"between [{datamodel_entity.name.root}, {str(upstream_datamart.targetDatamartId)}]: {exc}" # noqa: RUF010 + ), + stackTrace=traceback.format_exc(), + ) + ) + def yield_dashboard_lineage_details( self, dashboard_details: Group, @@ -2065,6 +2132,9 @@ def yield_dashboard_lineage_details( ) # 6. dataflow-upstreamDataflow lineage yield from self.create_dataflow_upstream_dataflow_lineage(datamodel, datamodel_entity) + elif isinstance(datamodel, Datamart): + # 7. datamart-upstreamDatamart lineage + yield from self.create_datamart_upstream_datamart_lineage(datamodel, datamodel_entity) else: logger.warning(f"Unknown datamodel type: {type(datamodel)}, name: {datamodel.name}") except Exception as exc: # pylint: disable=broad-except @@ -2145,6 +2215,8 @@ def get_owner_ref( # pylint: disable=unused-argument, useless-return # noqa: C access_right = owner.datasetUserAccessRight elif isinstance(dashboard_details, Dataflow): access_right = owner.dataflowUserAccessRight + elif isinstance(dashboard_details, Datamart): + access_right = owner.datamartUserAccessRight elif isinstance(dashboard_details, PowerBIReport): access_right = owner.reportUserAccessRight elif isinstance(dashboard_details, PowerBIDashboard): diff --git a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/models.py b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/models.py index 134f9bb3e6ca..b55183c4a3e3 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/models.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/models.py @@ -44,6 +44,7 @@ class PowerBIUser(BaseModel): reportUserAccessRight: Optional[str] = None # noqa: N815, UP045 datasetUserAccessRight: Optional[str] = None # noqa: N815, UP045 dataflowUserAccessRight: Optional[str] = None # noqa: N815, UP045 + datamartUserAccessRight: Optional[str] = None # noqa: N815, UP045 dashboardUserAccessRight: Optional[str] = None # noqa: N815, UP045 @@ -269,6 +270,27 @@ class Dataflow(BaseModel): upstreamDataflows: Optional[List[UpstreaDataflow]] = [] # noqa: N815, UP006, UP045 +class UpstreamDatamart(BaseModel): + groupId: Optional[str] = None # noqa: N815, UP045 + targetDatamartId: Optional[str] = None # noqa: N815, UP045 + + +class Datamart(BaseModel): + """ + PowerBI Datamart Model + Definition: https://learn.microsoft.com/en-us/rest/api/power-bi/admin/workspace-info-get-scan-result + """ + + id: str + name: str + description: Optional[str] = None # noqa: UP045 + type: Optional[str] = None # noqa: UP045 + users: Optional[List[PowerBIUser]] = [] # noqa: UP006, UP045 + configuredBy: Optional[str] = None # noqa: N815, UP045 + modifiedBy: Optional[str] = None # noqa: N815, UP045 + upstreamDatamarts: Optional[List[UpstreamDatamart]] = [] # noqa: N815, UP006, UP045 + + class Group(BaseModel): """ PowerBI Group Model @@ -283,6 +305,7 @@ class Group(BaseModel): reports: Optional[List[PowerBIReport]] = [] # noqa: UP006, UP045 datasets: Optional[List[Dataset]] = [] # noqa: UP006, UP045 dataflows: Optional[List[Dataflow]] = [] # noqa: UP006, UP045 + datamarts: Optional[List[Datamart]] = [] # noqa: UP006, UP045 class GroupsResponse(BaseModel): diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json index aa8e3a1a3518..21a361d6a646 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json @@ -29,6 +29,7 @@ "QuickSightDataModel", "SigmaDataModel", "PowerBIDataFlow", + "PowerBIDatamart", "MicroStrategyDataset", "ThoughtSpotDataModel", "SapS4HanaCdsView", @@ -71,6 +72,9 @@ { "name": "PowerBIDataFlow" }, + { + "name": "PowerBIDatamart" + }, { "name": "MicroStrategyDataset" }, diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDashboardDataModel.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDashboardDataModel.ts index 5c886d9ce5a5..21966000bf6f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDashboardDataModel.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDashboardDataModel.ts @@ -754,6 +754,7 @@ export enum DataModelType { MicroStrategyDataset = "MicroStrategyDataset", PowerBIDataFlow = "PowerBIDataFlow", PowerBIDataModel = "PowerBIDataModel", + PowerBIDatamart = "PowerBIDatamart", QlikDataModel = "QlikDataModel", QuickSightDataModel = "QuickSightDataModel", SapS4HanaCdsView = "SapS4HanaCdsView", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dashboardDataModel.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dashboardDataModel.ts index 46c6394d6bb9..fbf780dbcb0a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dashboardDataModel.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dashboardDataModel.ts @@ -898,6 +898,7 @@ export enum DataModelType { MicroStrategyDataset = "MicroStrategyDataset", PowerBIDataFlow = "PowerBIDataFlow", PowerBIDataModel = "PowerBIDataModel", + PowerBIDatamart = "PowerBIDatamart", QlikDataModel = "QlikDataModel", QuickSightDataModel = "QuickSightDataModel", SapS4HanaCdsView = "SapS4HanaCdsView",