diff --git a/ckanext/validate/assets/css/validate.css b/ckanext/validate/assets/css/validate.css index 0bd51da..bd31409 100644 --- a/ckanext/validate/assets/css/validate.css +++ b/ckanext/validate/assets/css/validate.css @@ -1,3 +1,4 @@ +/* === Badges de estado de validación (global) === */ .validate-badge { display: inline-block; padding: 0.25em 0.6em; @@ -46,6 +47,7 @@ } +/* === Página de validación de recurso (resource_validate.html) === */ .validate-page { max-width: 980px; } @@ -282,4 +284,142 @@ .validate-rules-details__content ul { margin-bottom: 10px; padding-left: 20px; -} \ No newline at end of file +} + +/* === Página de estadísticas de validación (validation_statistics.html) === */ +.validation-chart-card { + margin-bottom: 24px; + padding: 20px; + border: 1px solid #ddd; + border-radius: 4px; + background: #fff; +} + +.validation-chart-card h2 { + margin-top: 0; +} + +.validation-status-chart { + display: flex; + width: 100%; + height: 34px; + margin: 15px 0 10px; + overflow: hidden; + border-radius: 4px; + background: #eee; +} + +.validation-status-chart__valid { + background: #2e8540; +} + +.validation-status-chart__invalid { + background: #d9534f; +} + +.validation-status-chart__other { + background: #777; +} + +.validation-chart-legend { + display: flex; + flex-wrap: wrap; + gap: 18px; + margin: 0; + padding: 0; + list-style: none; +} + +.validation-chart-legend__marker { + display: inline-block; + width: 12px; + height: 12px; + margin-right: 5px; + border-radius: 2px; + vertical-align: -1px; +} + +.validation-chart-legend__marker--valid { + background: #2e8540; +} + +.validation-chart-legend__marker--invalid { + background: #d9534f; +} + +.validation-chart-legend__marker--other { + background: #777; +} + +.validation-horizontal-chart { + margin: 0; + padding: 0; + list-style: none; +} + +.validation-horizontal-chart__item { + display: grid; + grid-template-columns: minmax(160px, 1fr) 3fr 70px; + align-items: center; + gap: 12px; + margin-bottom: 12px; +} + +.validation-horizontal-chart__track { + height: 18px; + overflow: hidden; + border-radius: 3px; + background: #eee; +} + +.validation-horizontal-chart__bar { + min-width: 3px; + height: 100%; + background: #337ab7; +} + +.validation-timeline { + display: flex; + align-items: flex-end; + gap: 7px; + min-height: 230px; + padding: 15px 10px 0; + overflow-x: auto; + border-bottom: 1px solid #bbb; +} + +.validation-timeline__item { + display: flex; + flex: 1 0 28px; + flex-direction: column; + justify-content: flex-end; + min-width: 28px; + height: 210px; + text-align: center; +} + +.validation-timeline__value { + margin-bottom: 4px; + font-size: 11px; +} + +.validation-timeline__bar { + min-height: 2px; + border-radius: 3px 3px 0 0; + background: #337ab7; +} + +.validation-timeline__label { + margin-top: 6px; + font-size: 10px; + white-space: nowrap; + transform: rotate(-45deg); + transform-origin: top center; +} + +@media (max-width: 767px) { + .validation-horizontal-chart__item { + grid-template-columns: 1fr; + gap: 4px; + } +} diff --git a/ckanext/validate/blueprints/admin.py b/ckanext/validate/blueprints/admin.py index 99862e5..67615cd 100644 --- a/ckanext/validate/blueprints/admin.py +++ b/ckanext/validate/blueprints/admin.py @@ -1,10 +1,12 @@ import logging +from datetime import datetime, timezone from flask import Blueprint from ckan.lib import base from ckan.plugins import toolkit +from ckanext.validate.model.validation import Validation from ckanext.validate.model.validation_jobs import JobStatus, ValidationJob from ckanext.validate.helpers import format_timestamp_for_display @@ -84,3 +86,101 @@ def _enrich(job_dict): "selected_status": selected_status, }, ) + + +VALIDATION_STATISTICS_PERIODS = { + "1_month": "1 month", + "6_months": "6 months", + "1_year": "1 year", +} + + +@validation_jobs_blueprint.route( + "/ckan-admin/validation-statistics", + methods=["GET"], +) +def validation_statistics(): + """Display validation statistics for a predefined reporting period.""" + context = {"user": toolkit.current_user.name} + + try: + toolkit.check_access("sysadmin", context) + except toolkit.NotAuthorized: + base.abort( + 403, + toolkit._("Need to be system administrator to administer"), + ) + + selected_period = toolkit.request.args.get( + "period", + "1_month", + ).strip() + + if selected_period not in VALIDATION_STATISTICS_PERIODS: + toolkit.h.flash_error( + toolkit._("Invalid statistics period: {0}").format( + selected_period + ) + ) + selected_period = "1_month" + + report_end = datetime.now(timezone.utc) + validations = Validation.get_by_period( + selected_period, + end_date=report_end, + ) + summary = Validation.get_statistics_summary(validations) + errors_by_type = Validation.group_errors_by_type(validations) + resources = Validation.group_errors_by_resource(validations) + timeline = Validation.get_statistics_timeline( + validations, + selected_period, + end_date=report_end, + ) + max_error_type_count = max( + (item["count"] for item in errors_by_type), + default=1, + ) + max_timeline_error_count = max( + (item["error_count"] for item in timeline), + default=1, + ) + + resource_show = toolkit.get_action("resource_show") + + for resource_summary in resources: + resource_summary["resource_name"] = resource_summary["resource_id"] + resource_summary["resource_url"] = None + + try: + resource = resource_show( + context, + {"id": resource_summary["resource_id"]}, + ) + except (toolkit.ObjectNotFound, toolkit.NotAuthorized): + continue + + resource_summary["resource_name"] = ( + resource.get("name") + or resource.get("description") + or resource_summary["resource_id"] + ) + resource_summary["resource_url"] = toolkit.url_for( + "resource.read", + id=resource["package_id"], + resource_id=resource["id"], + ) + + return base.render( + "admin/validation_statistics.html", + extra_vars={ + "summary": summary, + "errors_by_type": errors_by_type, + "resources": resources, + "periods": VALIDATION_STATISTICS_PERIODS, + "selected_period": selected_period, + "timeline": timeline, + "max_error_type_count": max_error_type_count, + "max_timeline_error_count": max_timeline_error_count, + }, + ) diff --git a/ckanext/validate/model/validation.py b/ckanext/validate/model/validation.py index 2bb3bb6..a9ebde6 100644 --- a/ckanext/validate/model/validation.py +++ b/ckanext/validate/model/validation.py @@ -1,5 +1,7 @@ +from collections import Counter from datetime import datetime, timezone +from dateutil.relativedelta import relativedelta from sqlalchemy import Column, DateTime, Index, Integer, UnicodeText from ckan.model.types import JsonDictType @@ -8,6 +10,13 @@ from ckan.plugins import toolkit +VALIDATION_PERIODS = { + "1_month": 1, + "6_months": 6, + "1_year": 12, +} + + class Validation(toolkit.BaseModel, ActiveRecordMixin): """Stores the result of each Frictionless validation run for a resource.""" @@ -50,6 +59,276 @@ def get_latest(cls, resource_id): .first() ) + @classmethod + def get_by_date_range(cls, start_date, end_date): + """Return validations created within a date range. + + ``start_date`` is inclusive and ``end_date`` is exclusive. This avoids + returning the same validation in two consecutive reporting periods. + """ + if ( + not isinstance(start_date, datetime) + or not isinstance(end_date, datetime) + ): + raise ValueError("start_date and end_date must be datetime objects") + + if start_date >= end_date: + raise ValueError("start_date must be earlier than end_date") + + return ( + Session.query(cls) + .filter( + cls.created >= start_date, + cls.created < end_date, + ) + .order_by(cls.created.desc()) + .all() + ) + + @classmethod + def get_by_period(cls, period, end_date=None): + """Return validations from one of the supported reporting periods.""" + months = VALIDATION_PERIODS.get(period) + if months is None: + raise ValueError( + "Invalid period. Valid values are: {0}".format( + ", ".join(VALIDATION_PERIODS) + ) + ) + + end_date = end_date or datetime.now(timezone.utc) + if not isinstance(end_date, datetime): + raise ValueError("end_date must be a datetime object") + + start_date = end_date - relativedelta(months=months) + return cls.get_by_date_range(start_date, end_date) + + @staticmethod + def _get_error_type(error): + """Return a stable grouping key for a stored validation error.""" + if not isinstance(error, dict): + return "unknown" + + return ( + error.get("type") + or error.get("title") + or error.get("message") + or "unknown" + ) + + @classmethod + def group_errors_by_type(cls, validations): + """Count stored errors by error type for the given validations.""" + error_counts = Counter() + + for validation in validations: + errors = ( + validation.errors + if isinstance(validation.errors, list) + else [] + ) + + for error in errors: + error_counts[cls._get_error_type(error)] += 1 + + missing_details = max( + (validation.error_count or 0) - len(errors), + 0, + ) + if missing_details: + error_counts["unknown"] += missing_details + + return [ + {"type": error_type, "count": count} + for error_type, count in sorted( + error_counts.items(), + key=lambda item: (-item[1], item[0]), + ) + ] + + @classmethod + def group_errors_by_resource(cls, validations): + """Summarize validations and errors for each resource.""" + resources = {} + + for validation in validations: + summary = resources.setdefault( + validation.resource_id, + { + "resource_id": validation.resource_id, + "validation_count": 0, + "valid_count": 0, + "invalid_count": 0, + "error_count": 0, + "errors_by_type": Counter(), + }, + ) + + summary["validation_count"] += 1 + summary["error_count"] += validation.error_count or 0 + + if validation.status == "success": + summary["valid_count"] += 1 + elif validation.status == "failure": + summary["invalid_count"] += 1 + + errors = ( + validation.errors + if isinstance(validation.errors, list) + else [] + ) + + for error in errors: + error_type = cls._get_error_type(error) + summary["errors_by_type"][error_type] += 1 + + missing_details = max( + (validation.error_count or 0) - len(errors), + 0, + ) + if missing_details: + summary["errors_by_type"]["unknown"] += missing_details + + result = [] + + for summary in resources.values(): + summary["errors_by_type"] = [ + {"type": error_type, "count": count} + for error_type, count in sorted( + summary["errors_by_type"].items(), + key=lambda item: (-item[1], item[0]), + ) + ] + result.append(summary) + + return sorted( + result, + key=lambda item: ( + -item["error_count"], + item["resource_id"], + ), + ) + + @classmethod + def get_statistics_summary(cls, validations): + """Return the main statistical indicators for validations. + + The counts represent validation executions. ``resource_count`` reports + how many different resources were included in the selected period. + """ + validations = list(validations) + total_validations = len(validations) + valid_count = sum( + validation.status == "success" + for validation in validations + ) + invalid_count = sum( + validation.status == "failure" + for validation in validations + ) + other_count = total_validations - valid_count - invalid_count + error_count = sum( + validation.error_count or 0 + for validation in validations + ) + resource_count = len( + {validation.resource_id for validation in validations} + ) + + def percentage(count): + if not total_validations: + return 0.0 + return round((count / total_validations) * 100, 2) + + return { + "validation_count": total_validations, + "resource_count": resource_count, + "valid_count": valid_count, + "invalid_count": invalid_count, + "other_count": other_count, + "error_count": error_count, + "valid_percentage": percentage(valid_count), + "invalid_percentage": percentage(invalid_count), + "other_percentage": percentage(other_count), + } + + @classmethod + def get_statistics_timeline(cls, validations, period, end_date=None): + """Group validation activity into chart-friendly time buckets. + + The one-month report is grouped by day. Longer reports are grouped + by calendar month. Empty periods are included so the chart keeps a + continuous timeline. + """ + months = VALIDATION_PERIODS.get(period) + if months is None: + raise ValueError( + "Invalid period. Valid values are: {0}".format( + ", ".join(VALIDATION_PERIODS) + ) + ) + + end_date = end_date or datetime.now(timezone.utc) + if not isinstance(end_date, datetime): + raise ValueError("end_date must be a datetime object") + + start_date = end_date - relativedelta(months=months) + buckets = {} + + if period == "1_month": + current = start_date.date() + final = end_date.date() + + while current <= final: + buckets[current] = { + "key": current.isoformat(), + "label": current.strftime("%d/%m"), + "validation_count": 0, + "valid_count": 0, + "invalid_count": 0, + "error_count": 0, + } + current += relativedelta(days=1) + + def bucket_key(created): + return created.date() + + else: + current = start_date.replace(day=1).date() + final = end_date.replace(day=1).date() + + while current <= final: + buckets[current] = { + "key": current.isoformat(), + "label": current.strftime("%m/%Y"), + "validation_count": 0, + "valid_count": 0, + "invalid_count": 0, + "error_count": 0, + } + current += relativedelta(months=1) + + def bucket_key(created): + return created.replace(day=1).date() + + for validation in validations: + if not validation.created: + continue + + bucket = buckets.get(bucket_key(validation.created)) + if bucket is None: + continue + + bucket["validation_count"] += 1 + bucket["error_count"] += validation.error_count or 0 + + if validation.status == "success": + bucket["valid_count"] += 1 + elif validation.status == "failure": + bucket["invalid_count"] += 1 + + return list(buckets.values()) + @classmethod def get_resource_status(cls, resource_id): """Return the most recent validation status for a resource, or None.""" diff --git a/ckanext/validate/templates/admin/base.html b/ckanext/validate/templates/admin/base.html index 7aca714..7c9d161 100644 --- a/ckanext/validate/templates/admin/base.html +++ b/ckanext/validate/templates/admin/base.html @@ -3,6 +3,7 @@ {% block content_primary_nav %} {{ super() }} {{ h.build_nav_icon('validate_admin.validation_jobs', _('Validation Jobs'), icon='tasks') }} + {{ h.build_nav_icon('validate_admin.validation_statistics', _('Validation Statistics'), icon='bar-chart') }} {% endblock %} {% block styles %} diff --git a/ckanext/validate/templates/admin/validation_statistics.html b/ckanext/validate/templates/admin/validation_statistics.html new file mode 100644 index 0000000..a8f58f9 --- /dev/null +++ b/ckanext/validate/templates/admin/validation_statistics.html @@ -0,0 +1,214 @@ +{% extends "admin/base.html" %} + +{% block subtitle %}{{ _('Validation Statistics') }}{% endblock %} + +{% block primary_content_inner %} +
+ {{ _('Summary of CSV validation results for the selected period.') }} +
+ + + +{{ _('No validations were found for the selected period.') }}
+ {% endif %} +{{ _('No validations were found for the selected period.') }}
+ {% endif %} +| {{ _('Resource') }} | +{{ _('Validations') }} | +{{ _('Valid') }} | +{{ _('Invalid') }} | +{{ _('Errors') }} | +
|---|---|---|---|---|
+ {% if resource.resource_url %}
+
+ {{ resource.resource_name }}
+
+ {% else %}
+ {{ resource.resource_name }}
+ {% endif %}
+ |
+ {{ resource.validation_count }} | +{{ resource.valid_count }} | +{{ resource.invalid_count }} | +{{ resource.error_count }} | +