render per-dataset validation reports on dynamical.org

_data/catalog.js

@@ -82,6 +82,12 @@ function reshapeStacCollection(collection) {
 
   const licenseLinks = (collection.links || []).filter((l) => l.rel === "license");
 
+  // Optional validation-report asset published by reformatters. Surface its
+  // href so the catalog page can offer a fallback raw-HTML link for datasets
+  // where the on-site rendering can't fetch the markdown source.
+  const validationAsset = (collection.assets || {})["validation_report"];
+  const validation_report_href = validationAsset ? validationAsset.href : null;
+
   return {
     license_md: licenseMd(licenseLinks),
     name: collection.title,
@@ -103,6 +109,7 @@ function reshapeStacCollection(collection) {
     dimensions,
     variables,
     notebooks,
+    validation_report_href,
   };
 }
 

_data/validationReports.js

@@ -0,0 +1,49 @@
+const fetch = require("@11ty/eleventy-fetch");
+
+const STAC_BASE_URL = process.env.STAC_BASE_URL || "https://stac.dynamical.org";
+const VALIDATION_BASE_URL =
+  process.env.VALIDATION_REPORTS_BASE_URL ||
+  "https://dataset-validation-reports.dynamical.org";
+const STAC_CACHE_DURATION = process.env.STAC_CACHE_DURATION || "1d";
+
+module.exports = async function () {
+  const rootCatalog = await fetch(`${STAC_BASE_URL}/catalog.json`, {
+    type: "json",
+    duration: STAC_CACHE_DURATION,
+  });
+
+  const childLinks = (rootCatalog.links || []).filter((l) => l.rel === "child");
+  const datasetIds = childLinks
+    .map((l) => {
+      const m = l.href.match(/\/([^/]+)\/collection\.json(?:$|\?|#)?$/);
+      return m ? m[1] : null;
+    })
+    .filter(Boolean);
+
+  const results = await Promise.all(
+    datasetIds.map(async (datasetId) => {
+      const baseUrl = `${VALIDATION_BASE_URL}/${datasetId}/latest/`;
+      const mdUrl = `${baseUrl}validation_summary.md`;
+      try {
+        const markdown = await fetch(mdUrl, {
+          type: "text",
+          duration: STAC_CACHE_DURATION,
+        });
+        return { datasetId, baseUrl, markdown };
+      } catch (err) {
+        // 404s are expected — not every dataset has a published validation
+        // report yet. Surface any other failure so build errors don't get
+        // silently swallowed (network, DNS, 5xx, etc).
+        if (/\b404\b/.test(String(err && err.message))) return null;
+        console.warn(
+          `[validationReports] ${datasetId}: failed to fetch summary — ${err.message}`,
+        );
+        return null;
+      }
+    }),
+  );
+
+  const entries = results.filter(Boolean);
+  entries.sort((a, b) => a.datasetId.localeCompare(b.datasetId));
+  return { entries };
+};

_includes/base.njk

@@ -12,7 +12,7 @@
     <meta property="og:type" content="website"/>
     <meta property="og:title" content="dynamical.org"/>
     <meta property="og:site_name" content="dynamical.org"/>
-    <meta property="og:type" content="website"/>
+    <meta property="og:image" content="https://assets.dynamical.org/identity/logo/icon/icon-1024.png"/>
     <link rel="stylesheet" type="text/css" href="/main.css"/>
     <link rel="preconnect" href="https://fonts.googleapis.com"/>
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin/>

content/catalog-pages.njk

@@ -64,6 +64,12 @@ eleventyComputed:
   </table>
   <p>
     <span><a href="https://stac.dynamical.org/{{ entry.dataset_id }}/collection.json">STAC</a> (<a href="https://radiantearth.github.io/stac-browser/#/external/stac.dynamical.org/{{ entry.dataset_id }}/collection.json">browse</a>)</span>
+    {%- set validationReport = validationReports.entries | find('datasetId', entry.dataset_id) %}
+    {% if validationReport %}
+      · <a href="/catalog/{{ entry.dataset_id }}/validation/">validation report</a>
+    {% elif entry.validation_report_href %}
+      · <a href="{{ entry.validation_report_href }}">validation report</a>
+    {% endif %}
   </p>
   {% if model.description %}
     <p style="text-wrap: balance">{{ model.description | markdown | safe }}</p>

content/catalog/validation.11ty.js

@@ -0,0 +1,177 @@
+// Per-dataset validation report page. Renders validation_summary.md
+// (published by dynamical-org/reformatters to R2) into a near-identical
+// layout to the standalone HTML report at
+// https://dataset-validation-reports.dynamical.org/<dataset-id>/latest/validation_report.html.
+//
+// Mirrors reformatters' src/scripts/validation/render.py output — the
+// same post-process transforms over markdown-it's HTML, plus per-
+// variable plot injection. Image refs in the markdown are bare
+// filenames and get resolved against the report's baseUrl so the
+// rendered page can load them directly from R2.
+//
+// Page chrome and the scroll-spy TOC come from base.njk + main.css +
+// lib/markdown-toc.js. Everything in this file is validation-specific.
+
+const MarkdownIt = require("markdown-it");
+const markdownToc = require("../../lib/markdown-toc.js");
+
+const md = new MarkdownIt("commonmark").enable("table");
+
+function escapeAttr(s) {
+  return s.replace(/&/g, "&amp;").replace(/"/g, "&quot;");
+}
+
+// Per-variable heading rows in the markdown are `### \`name\``, which
+// markdown-it renders as `<h3><code>name</code></h3>`. For each one,
+// append a `<div class="plots">` block of the three R2 plots that share
+// the variable's name as a filename suffix.
+function injectVariablePlots(html) {
+  const re = /(<h3 id="[^"]*"><code>([^<]+)<\/code><\/h3>)([\s\S]*?)(?=<h[23][\s>]|$)/g;
+  return html.replace(re, (_full, heading, varName, body) => {
+    const v = varName;
+    const plots =
+      `<div class="plots">` +
+      `<a href="nulls_${v}.png" target="_blank">` +
+      `<img src="nulls_${v}.png" alt="${escapeAttr(v)} — null fraction"></a>` +
+      `<a href="spatial_${v}.png" target="_blank">` +
+      `<img src="spatial_${v}.png" alt="${escapeAttr(v)} — spatial comparison"></a>` +
+      `<a href="temporal_${v}.png" target="_blank">` +
+      `<img src="temporal_${v}.png" alt="${escapeAttr(v)} — time series comparison"></a>` +
+      `</div>`;
+    return `${heading}${body}${plots}`;
+  });
+}
+
+function pngLinksOpenInNewTab(html) {
+  return html.replace(/<a href="([^"]+\.png)">/g, '<a href="$1" target="_blank">');
+}
+
+// Apply the site's .table-container + .data table classes so markdown
+// tables match the catalog page. The site-wide .table-container rule
+// lets tables overflow their parent into the page margins when the
+// viewport has room and only scrolls horizontally when the viewport is
+// narrower than the table.
+function styleTables(html) {
+  return html.replace(
+    /<table>([\s\S]*?)<\/table>/g,
+    '<div class="table-container"><table class="data">$1</table></div>',
+  );
+}
+
+// The page generates its own H1 from the catalog entry name, so any H1
+// emitted by the rendered markdown is redundant. reformatters PR #614
+// drops the H1 from validation_summary.md; this strip keeps the page
+// looking right against both the current bucket output (still includes
+// the H1) and the post-#614 output (no H1).
+function stripH1(html) {
+  return html.replace(/<h1>[\s\S]*?<\/h1>\s*/g, "");
+}
+
+// Resolve bare-filename src/href on <img>/<a> against baseUrl. Anything
+// that already looks absolute (scheme, protocol-relative, root-relative,
+// or a bare fragment) is left alone — only relative refs get rewritten.
+function rewriteAssetUrls(html, baseUrl) {
+  const isAbsolute = (u) =>
+    /^[a-z][a-z0-9+.-]*:/i.test(u) ||
+    u.startsWith("//") ||
+    u.startsWith("/") ||
+    u.startsWith("#");
+  const rewriteAttr = (input, attr) =>
+    input.replace(
+      new RegExp(`(<(?:img|a)\\b[^>]*\\s${attr}=")([^"]+)(")`, "g"),
+      (full, pre, url, post) =>
+        isAbsolute(url) ? full : `${pre}${baseUrl}${url}${post}`,
+    );
+  return rewriteAttr(rewriteAttr(html, "src"), "href");
+}
+
+function extractDatasetName(mdText, fallback) {
+  const m = mdText.match(/^\|\s*Validation\s*\|\s*([^|]+?)\s*\|/m);
+  return m ? m[1] : fallback;
+}
+
+// Validation-specific styles only: the per-variable plot block under
+// each <h3><code>name</code></h3>. Layout (wrapper, toc rail, sticky,
+// narrow-mode collapse) comes from lib/markdown-toc CSS; everything
+// else (max-width, typography, tables, link colors) comes from
+// main.css's .content + table.data + base type rules.
+const CSS = `
+.plots {
+  display: flex; flex-direction: column;
+  gap: 1rem; margin: 1rem 0 2rem;
+}
+.plots a { display: block; }
+.plots img {
+  display: block; max-width: 100%; height: auto;
+  border: 1px solid var(--border-color);
+  background: var(--bg-color);
+}
+${markdownToc.CSS}
+`;
+
+function renderFragment({ datasetId, baseUrl, markdown }, datasetName) {
+  let html = md.render(markdown);
+  html = stripH1(html);
+  html = pngLinksOpenInNewTab(html);
+
+  const annotated = markdownToc.annotateHeadings(html);
+  html = annotated.html;
+
+  html = injectVariablePlots(html);
+  html = styleTables(html);
+  html = rewriteAssetUrls(html, baseUrl);
+
+  const tocHtml = markdownToc.buildTocHtml(annotated.headings);
+  const breadcrumbName = datasetName || datasetId;
+  const pageTitle = `${breadcrumbName} validation report`;
+
+  // .content gives the standard 78rem max-width + centering. The TOC
+  // rail absolute-positions itself out of flow into the left margin
+  // (containing block = .md-toc-content). On narrow viewports the rail
+  // goes static and renders here in flow, under the H1.
+  return `<div class="content md-toc-content">
+  <div>
+    <a href="/catalog">Catalog</a> >
+    <a href="/catalog/${datasetId}/">${breadcrumbName}</a> >
+    Validation report
+  </div>
+  <h1>${pageTitle}</h1>
+  <div class="md-toc-rail">
+    <nav class="md-toc" aria-label="Table of contents">${tocHtml}</nav>
+  </div>
+  ${html}
+  <style>${CSS}</style>
+  <script>${markdownToc.JS}</script>
+</div>`;
+}
+
+class ValidationReportPage {
+  data() {
+    return {
+      layout: "base.njk",
+      pagination: {
+        data: "validationReports.entries",
+        size: 1,
+        alias: "entry",
+      },
+      permalink: ({ entry }) => `catalog/${entry.datasetId}/validation/`,
+      eleventyComputed: {
+        title: ({ entry }) => `Validation report — ${entry.datasetId}`,
+      },
+    };
+  }
+
+  render({ entry, catalog }) {
+    const catalogEntry =
+      catalog && catalog.entries
+        ? catalog.entries.find((e) => e.dataset_id === entry.datasetId)
+        : null;
+    const datasetName =
+      (catalogEntry && catalogEntry.name) ||
+      extractDatasetName(entry.markdown, entry.datasetId);
+    return renderFragment(entry, datasetName);
+  }
+}
+
+module.exports = ValidationReportPage;
+module.exports.renderFragment = renderFragment;