render per-dataset validation reports on dynamical.org

_data/catalog.js

@@ -82,6 +82,12 @@ function reshapeStacCollection(collection) {
 
   const licenseLinks = (collection.links || []).filter((l) => l.rel === "license");
 
+  // Optional validation-report asset published by reformatters. Surface its
+  // href so the catalog page can offer a fallback raw-HTML link for datasets
+  // where the on-site rendering can't fetch the markdown source.
+  const validationAsset = (collection.assets || {})["validation_report"];
+  const validation_report_href = validationAsset ? validationAsset.href : null;
+
   return {
     license_md: licenseMd(licenseLinks),
     name: collection.title,
@@ -103,6 +109,7 @@ function reshapeStacCollection(collection) {
     dimensions,
     variables,
     notebooks,
+    validation_report_href,
   };
 }
 

_data/validationReports.js

@@ -0,0 +1,49 @@
+const fetch = require("@11ty/eleventy-fetch");
+
+const STAC_BASE_URL = process.env.STAC_BASE_URL || "https://stac.dynamical.org";
+const VALIDATION_BASE_URL =
+  process.env.VALIDATION_REPORTS_BASE_URL ||
+  "https://dataset-validation-reports.dynamical.org";
+const STAC_CACHE_DURATION = process.env.STAC_CACHE_DURATION || "1d";
+
+module.exports = async function () {
+  const rootCatalog = await fetch(`${STAC_BASE_URL}/catalog.json`, {
+    type: "json",
+    duration: STAC_CACHE_DURATION,
+  });
+
+  const childLinks = (rootCatalog.links || []).filter((l) => l.rel === "child");
+  const datasetIds = childLinks
+    .map((l) => {
+      const m = l.href.match(/\/([^/]+)\/collection\.json(?:$|\?|#)?$/);
+      return m ? m[1] : null;
+    })
+    .filter(Boolean);
+
+  const results = await Promise.all(
+    datasetIds.map(async (datasetId) => {
+      const baseUrl = `${VALIDATION_BASE_URL}/${datasetId}/latest/`;
+      const mdUrl = `${baseUrl}validation_summary.md`;
+      try {
+        const markdown = await fetch(mdUrl, {
+          type: "text",
+          duration: STAC_CACHE_DURATION,
+        });
+        return { datasetId, baseUrl, markdown };
+      } catch (err) {
+        // 404s are expected — not every dataset has a published validation
+        // report yet. Surface any other failure so build errors don't get
+        // silently swallowed (network, DNS, 5xx, etc).
+        if (/\b404\b/.test(String(err && err.message))) return null;
+        console.warn(
+          `[validationReports] ${datasetId}: failed to fetch summary — ${err.message}`,
+        );
+        return null;
+      }
+    }),
+  );
+
+  const entries = results.filter(Boolean);
+  entries.sort((a, b) => a.datasetId.localeCompare(b.datasetId));
+  return { entries };
+};

_includes/base.njk

@@ -12,7 +12,7 @@
     <meta property="og:type" content="website"/>
     <meta property="og:title" content="dynamical.org"/>
     <meta property="og:site_name" content="dynamical.org"/>
-    <meta property="og:type" content="website"/>
+    <meta property="og:image" content="https://assets.dynamical.org/identity/logo/icon/icon-1024.png"/>
     <link rel="stylesheet" type="text/css" href="/main.css"/>
     <link rel="preconnect" href="https://fonts.googleapis.com"/>
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin/>

content/catalog-pages.njk

@@ -64,6 +64,13 @@ eleventyComputed:
   </table>
   <p>
     <span><a href="https://stac.dynamical.org/{{ entry.dataset_id }}/collection.json">STAC</a> (<a href="https://radiantearth.github.io/stac-browser/#/external/stac.dynamical.org/{{ entry.dataset_id }}/collection.json">browse</a>)</span>
+    {%- set validationReport = validationReports.entries | find('datasetId', entry.dataset_id) %}
+    {% if validationReport %}
+      · <a href="/catalog/{{ entry.dataset_id }}/validation/">validation report</a>
+      (<a href="{{ validationReport.baseUrl }}validation_report.html">raw</a>)
+    {% elif entry.validation_report_href %}
+      · <a href="{{ entry.validation_report_href }}">validation report</a>
+    {% endif %}
   </p>
   {% if model.description %}
     <p style="text-wrap: balance">{{ model.description | markdown | safe }}</p>

content/catalog/validation.11ty.js

@@ -0,0 +1,227 @@
+// Per-dataset validation report page. Renders validation_summary.md
+// (published by dynamical-org/reformatters to R2) into a near-identical
+// layout to the standalone HTML report at
+// https://dataset-validation-reports.dynamical.org/<dataset-id>/latest/validation_report.html.
+//
+// Mirrors reformatters' src/scripts/validation/render.py output — the
+// same post-process transforms over markdown-it's HTML, plus per-
+// variable plot injection. Image refs in the markdown are bare
+// filenames and get resolved against the report's baseUrl so the
+// rendered page can load them directly from R2.
+//
+// Page chrome and the scroll-spy TOC come from base.njk + main.css +
+// lib/markdown-toc.js. Everything in this file is validation-specific.
+
+const MarkdownIt = require("markdown-it");
+const markdownToc = require("../../lib/markdown-toc.js");
+
+const md = new MarkdownIt("commonmark").enable("table");
+
+function escapeAttr(s) {
+  return s.replace(/&/g, "&amp;").replace(/"/g, "&quot;");
+}
+
+// Per-variable heading rows in the markdown are `### \`name\``, which
+// markdown-it renders as `<h3><code>name</code></h3>`. For each one,
+// append a `<div class="plots">` block of the three R2 plots that share
+// the variable's name as a filename suffix.
+function injectVariablePlots(html) {
+  const re = /(<h3 id="[^"]*"><code>([^<]+)<\/code><\/h3>)([\s\S]*?)(?=<h[23][\s>]|$)/g;
+  return html.replace(re, (_full, heading, varName, body) => {
+    const v = varName;
+    const plots =
+      `<div class="plots">` +
+      `<a href="nulls_${v}.png" target="_blank">` +
+      `<img src="nulls_${v}.png" alt="${escapeAttr(v)} — null fraction"></a>` +
+      `<a href="spatial_${v}.png" target="_blank">` +
+      `<img src="spatial_${v}.png" alt="${escapeAttr(v)} — spatial comparison"></a>` +
+      `<a href="temporal_${v}.png" target="_blank">` +
+      `<img src="temporal_${v}.png" alt="${escapeAttr(v)} — time series comparison"></a>` +
+      `</div>`;
+    return `${heading}${body}${plots}`;
+  });
+}
+
+function pngLinksOpenInNewTab(html) {
+  return html.replace(/<a href="([^"]+\.png)">/g, '<a href="$1" target="_blank">');
+}
+
+function wrapTables(html) {
+  return html.replace(
+    /(<table>[\s\S]*?<\/table>)/g,
+    '<div class="table-scroll">$1</div>',
+  );
+}
+
+// Resolve bare-filename src/href on <img>/<a> against baseUrl. Anything
+// that already looks absolute (scheme, protocol-relative, root-relative,
+// or a bare fragment) is left alone — only relative refs get rewritten.
+function rewriteAssetUrls(html, baseUrl) {
+  const isAbsolute = (u) =>
+    /^[a-z][a-z0-9+.-]*:/i.test(u) ||
+    u.startsWith("//") ||
+    u.startsWith("/") ||
+    u.startsWith("#");
+  const rewriteAttr = (input, attr) =>
+    input.replace(
+      new RegExp(`(<(?:img|a)\\b[^>]*\\s${attr}=")([^"]+)(")`, "g"),
+      (full, pre, url, post) =>
+        isAbsolute(url) ? full : `${pre}${baseUrl}${url}${post}`,
+    );
+  return rewriteAttr(rewriteAttr(html, "src"), "href");
+}
+
+function extractDatasetName(mdText, fallback) {
+  const m = mdText.match(/^\|\s*Validation\s*\|\s*([^|]+?)\s*\|/m);
+  return m ? m[1] : fallback;
+}
+
+// Layout: the report body is centered at the same max-width as the
+// rest of the site (78rem), unaffected by the TOC. The TOC lives in a
+// rail anchored just outside the wrapper's left edge (so it sits in
+// what would otherwise be empty page margin) and stays sticky as the
+// page scrolls. Below ~1180px the rail collapses back into the flow
+// and renders above the content.
+//
+// Typography, link colors, base table style come from main.css.
+// Nested-tree styling + ▸ indicator come from lib/markdown-toc CSS.
+const CSS = `
+.validation-wrapper {
+  position: relative;
+  max-width: 78rem;
+  margin: 0 auto;
+}
+.validation-toc-rail {
+  position: absolute;
+  top: 0;
+  right: calc(100% + 3rem);
+  width: 18rem;
+  height: 100%;
+}
+.md-toc {
+  position: sticky;
+  top: 2rem;
+  font-size: 1.2rem;
+  max-height: calc(100vh - 4rem);
+  overflow-y: auto;
+}
+
+.validation-body {
+  font-size: 1.4rem;
+}
+.validation-breadcrumb { margin-bottom: 2rem; }
+.validation-body h2 { margin-top: 3.2rem; }
+.validation-body h3 { margin-top: 2.4rem; }
+.validation-body .plots {
+  display: flex; flex-direction: column;
+  gap: 1rem; margin: 1rem 0 2rem;
+}
+.validation-body .plots a { display: block; }
+.validation-body .plots img {
+  display: block; max-width: 100%; height: auto;
+  border: 1px solid var(--border-color);
+  background: var(--bg-color);
+}
+.validation-body .table-scroll {
+  overflow-x: auto; margin: 1rem 0 2rem;
+}
+.validation-body .table-scroll table {
+  border-collapse: collapse;
+  border: 1px solid var(--border-color);
+  margin: 0;
+  max-width: 100%;
+}
+.validation-body .table-scroll th,
+.validation-body .table-scroll td {
+  padding: 0.8rem 1.6rem;
+  vertical-align: top;
+  border-right: 1px dotted var(--border-muted-color);
+}
+.validation-body .table-scroll th {
+  border-bottom: 1px solid var(--border-color);
+  font-weight: 700;
+}
+
+@media (max-width: 1180px) {
+  .validation-toc-rail {
+    position: static;
+    width: auto;
+    height: auto;
+    margin-bottom: 1.6rem;
+  }
+  .md-toc {
+    position: static;
+    max-height: none;
+    overflow: visible;
+    padding: 1rem 0;
+  }
+  .validation-body .table-scroll table { font-size: 1.2rem; }
+  .validation-body .table-scroll th,
+  .validation-body .table-scroll td {
+    padding: 0.4rem 0.8rem;
+  }
+}
+${markdownToc.CSS}
+`;
+
+function renderFragment({ datasetId, baseUrl, markdown }, datasetName) {
+  let html = md.render(markdown);
+  html = pngLinksOpenInNewTab(html);
+
+  const annotated = markdownToc.annotateHeadings(html);
+  html = annotated.html;
+
+  html = injectVariablePlots(html);
+  html = wrapTables(html);
+  html = rewriteAssetUrls(html, baseUrl);
+
+  const tocHtml = markdownToc.buildTocHtml(annotated.headings);
+  const breadcrumbName = datasetName || datasetId;
+
+  return `<div class="validation-wrapper">
+  <div class="validation-toc-rail">
+    <nav class="md-toc" aria-label="Table of contents">${tocHtml}</nav>
+  </div>
+  <div class="md-toc-content validation-body">
+    <div class="validation-breadcrumb">
+      <a href="/catalog">Catalog</a> >
+      <a href="/catalog/${datasetId}/">${breadcrumbName}</a> >
+      Validation report
+    </div>
+    ${html}
+  </div>
+  <style>${CSS}</style>
+  <script>${markdownToc.JS}</script>
+</div>`;
+}
+
+class ValidationReportPage {
+  data() {
+    return {
+      layout: "base.njk",
+      pagination: {
+        data: "validationReports.entries",
+        size: 1,
+        alias: "entry",
+      },
+      permalink: ({ entry }) => `catalog/${entry.datasetId}/validation/`,
+      eleventyComputed: {
+        title: ({ entry }) => `Validation report — ${entry.datasetId}`,
+      },
+    };
+  }
+
+  render({ entry, catalog }) {
+    const catalogEntry =
+      catalog && catalog.entries
+        ? catalog.entries.find((e) => e.dataset_id === entry.datasetId)
+        : null;
+    const datasetName =
+      (catalogEntry && catalogEntry.name) ||
+      extractDatasetName(entry.markdown, entry.datasetId);
+    return renderFragment(entry, datasetName);
+  }
+}
+
+module.exports = ValidationReportPage;
+module.exports.renderFragment = renderFragment;