diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6546f23..2e28575 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,13 +2,17 @@ name: Python Unit Tests on: push: - branches: [ "main" ] + branches: + - '**' # Matches every branch (feature branches, main, etc.) pull_request: - branches: [ "main" ] + branches: + - '**' # Runs tests for PRs targeting any branch jobs: test: runs-on: ubuntu-latest + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' steps: - name: Checkout code @@ -19,11 +23,12 @@ jobs: with: python-version: '3.10' - - name: Install dependencies + - name: Install System Dependencies run: | - python -m pip install --upgrade pip - pip install pytest click numpy + sudo apt-get update + sudo apt-get install -y libudunits2-dev # This is the missing piece - - name: Run Pytest + - name: Install Python Dependencies run: | - pytest + python -m pip install --upgrade pip + pip install pytest click numpy cf-python cftime diff --git a/.gitignore b/.gitignore index aa73532..8075e19 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ missing* .DS_Store .ipynb_checkpoints +*.ipynb .nfs* .html __pycache__ @@ -40,3 +41,5 @@ docs/*.aux docs/*.log docs/*.toc docs/*.synctex.gz + +*failed*.txt diff --git a/docs/Technical_Report_CFA_CANARI.pdf b/docs/Technical_Report_CFA_CANARI.pdf index 452a0d5..58abcca 100644 Binary files a/docs/Technical_Report_CFA_CANARI.pdf and b/docs/Technical_Report_CFA_CANARI.pdf differ diff --git a/docs/Technical_Report_CFA_CANARI.tex b/docs/Technical_Report_CFA_CANARI.tex index 156914a..0e8d8b8 100644 --- a/docs/Technical_Report_CFA_CANARI.tex +++ b/docs/Technical_Report_CFA_CANARI.tex @@ -2,12 +2,25 @@ \usepackage[margin=1in]{geometry} \usepackage{graphicx} -\usepackage{xcolor} -\usepackage[hidelinks]{hyperref} + + +\usepackage[dvipsnames]{xcolor} % Gives you access to better color names +\usepackage{hyperref} + +\hypersetup{ + colorlinks=true, % Turns off the boxes, turns on colored text + linkcolor=RoyalBlue, % Color for internal links (Figs, Tabs, Equations, TOC) + citecolor=ForestGreen, % Color for bibliographical citations + urlcolor=Bittersweet, % Color for external web links + filecolor=magenta, % Color for local file links + pdfpagemode=FullScreen, +} + \usepackage{listings} +\usepackage{gensymb} \usepackage{svg} -\usepackage{fancyvrb} -\usepackage{rotating} +\usepackage{fancyvrb} +\usepackage{rotating} \usepackage{microtype} \usepackage[title]{appendix} \usepackage{float} @@ -71,10 +84,10 @@ \thispagestyle{fancy} \tableofcontents - +\clearpage \section{Introduction} -This document details the processes by which data from the CANARI project -- large ensembles of simulations using the HadGEM3-GC3.1 climate model \citep{williams2018} -- has been aggregated after its ingestion into the Joint-storage Data Migration App (JDMA) archive\footnote{\url{https://help.jasmin.ac.uk/docs/short-term-project-storage/jdma/}}. The JDMA archive is hosted on the JASMIN platform hosted by the Centre for Environmental Data Analysis, CEDA\footnote{\url{https://www.ceda.ac.uk}}. +This document details the processes by which data from the CANARI project -- large ensembles of simulations using the HadGEM3-GC3.1 climate model \citep{williams2018} -- has been aggregated after its ingestion into the Joint-storage Data Migration App \label{jdma} (JDMA) archive\footnote{\url{https://help.jasmin.ac.uk/docs/short-term-project-storage/jdma/}}. The JDMA archive is hosted on the JASMIN platform hosted by the Centre for Environmental Data Analysis, CEDA\footnote{\url{https://www.ceda.ac.uk}}. The CANARI ensemble consists of two, 40-member ensembles: @@ -194,7 +207,7 @@ \subsection{Seed}\label{sseed} \begin{figure} \centering - \includegraphics[width=\textwidth]{figures/seed.png} + \includegraphics[width=0.8\textwidth]{figures/seed.png} \caption{Algorithm for the \texttt{seed.py} file, prepared with \texttt{PlantUML} \citep{plantuml}. } \label{fig:seed} \end{figure} @@ -265,15 +278,16 @@ \subsubsection{JDMA batch IDs} \subsubsection{JDMA `external' IDs} -Ongoing work will see the data on the JDMA archive moved to the Near-Line Data Store (NLDS).\footnote{\url{https://help.jasmin.ac.uk/docs/short-term-project-storage/nlds/}} To assist in this process, the elastic tape `external IDs' are also required. These numbers are found from the \texttt{jdma batch} command as follows: +Ongoing work will see the data on the JDMA archive moved to the Near-Line Data Store (NLDS).\footnote{\url{https://help.jasmin.ac.uk/docs/short-term-project-storage/nlds/}} To assist in this process, the elastic tape `external IDs' are also required. These numbers are found from the \texttt{jdma batch} command as shown in Listing \ref{lst:jdma-batch}): \begin{center} \begin{minipage}{\textwidth} \begin{lstlisting}[ basicstyle=\small\ttfamily, + label={lst:jdma-batch}, frame=single, backgroundcolor=\color{gray!5}, - breaklines=true, + breaklines=true,caption={\texttt{jdma batch} output}, keepspaces=true, columns=fullflexible ] @@ -386,15 +400,16 @@ \section{Benchmarking}\label{bench} \subsection{Seed}\label{aseed} +Listing \ref{lst:seed} \begin{center} \begin{minipage}{\textwidth} \begin{lstlisting}[ basicstyle=\small\ttfamily, - frame=single, + frame=single,caption={\texttt{seed.py}}, backgroundcolor=\color{gray!5}, - breaklines=true, + breaklines=true,label={lst:seed}, keepspaces=true, columns=fullflexible ] @@ -433,14 +448,14 @@ \subsection{Seed}\label{aseed} \subsection{Grow} -As for \S~\ref{aseed} for the growth step. +As for \S~\ref{aseed} for the growth step. Code usage in Listing \ref{lst:grow}. \begin{center} \begin{minipage}{\textwidth} \begin{lstlisting}[ basicstyle=\small\ttfamily, - frame=single, - backgroundcolor=\color{gray!5}, + frame=single,caption={\texttt{grow.py}}, + backgroundcolor=\color{gray!5},label={lst:grow}, breaklines=true, keepspaces=true, columns=fullflexible @@ -476,13 +491,13 @@ \subsection{Grow} \subsection{Arrange} -As for \S~\ref{aseed} for the arrange step. +As for \S~\ref{aseed} for the arrange step. Code usage shown in Listing \ref{lst:arrange}. \begin{center} \begin{minipage}{\textwidth} \begin{lstlisting}[ basicstyle=\small\ttfamily, - frame=single, + frame=single,label={lst:arrange}, caption={\texttt{arrange.py}}, backgroundcolor=\color{gray!5}, breaklines=true, keepspaces=true, @@ -533,6 +548,8 @@ \section{Runtime instructions} \subsection{Seed} +Listing \ref{lst:aseed}. + \begin{minipage}{\textwidth} @@ -540,7 +557,7 @@ \subsection{Seed} caption={Output of python seed.py -h}, basicstyle=\small\ttfamily, frame=single, - breaklines=true, + breaklines=true,label={lst:aseed}, backgroundcolor=\color{gray!5}, columns=fullflexible, keepspaces=true @@ -570,12 +587,13 @@ \subsection{Seed} \subsection{Grow} +Listing \ref{lst:agrow}. \begin{lstlisting}[ caption={Output of python grow.py -h}, basicstyle=\small\ttfamily, frame=single, - breaklines=true, + breaklines=true,label={lst:agrow}, backgroundcolor=\color{gray!5}, columns=fullflexible, keepspaces=true @@ -599,11 +617,13 @@ \subsection{Grow} \subsection{Arrange} +Listing \ref{lst:aarrange}. + \begin{minipage}{\textwidth} \begin{lstlisting}[ caption={Output of python arrange.py -h}, basicstyle=\small\ttfamily, - frame=single, + frame=single,label={lst:aarrange}, breaklines=true, backgroundcolor=\color{gray!5}, columns=fullflexible, @@ -627,9 +647,327 @@ \subsection{Arrange} \end{minipage} +\section{Priority variables} + + +\subsection{Background} + +The entirety of this report has dealt with the CF aggregration of the entire CANARI output as stored (currently, May 2026) on the JDMA system. For many applications however, only a subset of data is required and this is referred to as the `priority variables' -- PVs -- set. Detailed information regarding the variables themselves can be found on the relevant \href{https://ncas-cms.github.io/canari/#priority-and-derived-output-on-jasmin-canari-group-workspace}{NCAS-CMS GitHub page}. + +\subsection{Algorithm} + +For the priority variables, all data is accessible `live' on the `canari' Group Workspace on JASMIN at `/gws/ssde/j25b/canari/large-ensemble/priority-variables`. In addition, the PVs are pre-separated into single variable files, for example (see Figure \ref{fig:ATMOCNCICE}): + +\begin{itemize} + + \item \texttt{\detokenize{cv575a_1_1hr_m01s05i216.nc}}, for the atmosphere. + \begin{itemize} + \item Surface temperature after timestep [\degree K], details of variables can be found on the \href{https://reference.metoffice.gov.uk/um/stash}{UM STASH browser}. + % \item For example see Figure \ref{fig:ATM}. +\end{itemize} + \item \texttt{\detokenize{cv575o_1_day__grid_T_tos.nc}}, for the ocean. + \begin{itemize} + \item Sea surface temperature [\degree C]. + % \item For example Figure \ref{fig:OCN} + \end{itemize} + \item \texttt{\detokenize{cv575i_1_day_aice.nc}}, for the sea ice. + \begin{itemize} + \item Ice area (aggregate) $ \in [0, 1]$. + % \item For example Figure \ref{fig:CICE} + + \end{itemize} + +\end{itemize} + + +Because of the availability of the PVs on disk, the process for obtaining the CFA files representing each realm is greatly simplified and is performed by a single Python script, \texttt{priority-variables.py}. This script reads in all the data for each member of the historical and SSP3-7.0 ensembles and writes out one CFA files for each variable/scenario/member combination. The \texttt{priority-variables} Python script which performs this is illustrated using \texttt{PlantUML} in Figure \ref{fig:priority-variables}. + + +\begin{figure} + \centering + \includegraphics[width=0.8\textwidth]{figures/priority-variables.png} + \caption{Algorithm for the \texttt{priority-files.py} file, Figure prepared using \texttt{PlantUML}. } + \label{fig:priority-variables} +\end{figure} + +\begin{minipage}{\textwidth} +\begin{lstlisting}[ + caption={Output of python priority-variables.py {-}{-}help}, + basicstyle=\small\ttfamily, + frame=single,label={lst:priority-variables}, + breaklines=true, + backgroundcolor=\color{gray!5}, + columns=fullflexible, + keepspaces=true +] +python priority-variables.py --help +Usage: priority-variables.py [OPTIONS] + + Create seed CFA file for CANARI priority variables. + +Options: + --realm [ATM|OCN|CICE] The climate realm [required] + --verbose INTEGER RANGE Verbosity level of cf.read [-1<=x<=3] + --scenario [HIST2|SSP370] The scenario [required] + -m, --member INTEGER The ensemble member number [required] + -d, --data_path PATH Path to the source data directory [required] + --help Show this message and exit. +(cfa_env) sci-vm-01|Wed May 27|15:44:32 BST|priority-variables|[priority-variables]> +\end{lstlisting} +\end{minipage} + + +\begin{figure}[htbp] + \centering + \includegraphics[width=0.8\textwidth]{figures/ATMOCNCICE} + \caption{Example surface air temperature after timestep (in Kelvin), sea surface temperature (in Celsius), and sea ice area (aggregate) from ensemble member 1 of the historical ensemble.} + \label{fig:ATMOCNCICE} +\end{figure} + +The code for generating Figure \ref{fig:ATMOCNCICE} is shown in Listing \ref{lst:ATMOCNCICE}. + + +\clearpage + + + +\begin{lstlisting}[ + caption={Code to generate Figure \ref{fig:ATMOCNCICE}.}, % Added backslash here + basicstyle=\small\ttfamily, + label={lst:ATMOCNCICE}, + frame=single, + breaklines=true, + backgroundcolor=\color{gray!5}, + columns=fullflexible, + keepspaces=true +] +import matplotlib.pyplot as plt +import cartopy.crs as ccrs +import cartopy.feature as cfeature +import cf + +files = [ + "./CF-1.13_seed_CANARI_1_cv575_ATM_1_mon_m01s00i024_3.cfa", + "./CF-1.13_seed_CANARI_1_cv575_OCN_day__grid_T_tos.cfa", + "CF-1.13_seed_CANARI_1_cv575_CICE_day_aice.cfa", +] + [ + "CF-1.13_seed_CANARI_1_cv575_CICE_day_aice.cfa", +] + +fig, axes = plt.subplots( + nrows=2, + ncols=2, + layout="constrained", +) + +if len(files) == 1: + axes = [axes] +else: + axes = axes.flatten() + +cmaps = ["RdBu_r"] * 2 + ["Blues"] * 2 + +for i in range(len(files)): + old_ax = axes[i] + pos = old_ax.get_subplotspec() + old_ax.remove() + + if i < 2: + proj = ccrs.Robinson() + elif i == 2: + proj = ccrs.SouthPolarStereo() + elif i == 3: + proj = ccrs.NorthPolarStereo() + + ax = fig.add_subplot(pos, projection=proj) + + fields = cf.read(files[i]) + f = fields[0][0] + + mesh = ax.pcolormesh( + f.construct("X").data.array, + f.construct("Y").data.array, + f.data.array.squeeze(), + transform=ccrs.PlateCarree(), + shading="auto", + cmap=cmaps[i], + zorder=0, + ) + + ax.coastlines(resolution="110m", linewidth=1) + + if i == 2: + ax.set_extent([-180, 180, -90 * (i == 2), -60 * (i == 2)], ccrs.PlateCarree()) + elif i == 3: + ax.set_extent([-180, 180, 60, 90], ccrs.PlateCarree()) + else: + ax.set_global() + + if i > 0: + ax.add_feature(cfeature.LAND, zorder=1) + + plt.colorbar(mesh, ax=ax, orientation="vertical", pad=0.02, shrink=0.6) + + title_text = getattr(f, "long_name", f"File {i}") + ax.set_title(title_text) + +plt.show(); + +\end{lstlisting} + + + +\clearpage + +\subsection{Discovery and generation of single-variable files} + +The full CANARI data set is held on the JDMA system -- see page \pageref{jdma} -- but to retrieve individual variables there is a 2 step process: + +\begin{enumerate} + \item Retrieve monthly data from JDMA for the years(s) required (see Figure \ref{fig:jdma_get_monthly}). + \item Extract the required data and write to single-variable files (see Figure \ref{fig:get-yearly}). +\end{enumerate} + +\subsubsection{Retrieval} + +Figure \ref{fig:jdma_get_monthly} shows this graphically and Listing \ref{lst:jdma_get_monthly} gives the code implementation. + + +\begin{figure} + \centering + \includegraphics[width=0.8\textwidth]{figures/jdma_get_monthly.png} + \caption{Algorithm for the \texttt{\detokenize{jdma_get_monthly.py}} file, Figure prepared using \texttt{PlantUML}. } + \label{fig:jdma_get_monthly} +\end{figure} + +\begin{lstlisting}[ + caption={Output of python jdma\_get\_monthly.py {-}{-}help}, % Added backslash here + label={lst:jdma_get_monthly}, + basicstyle=\small\ttfamily, + frame=single, + breaklines=true, + backgroundcolor=\color{gray!5}, + columns=fullflexible, + keepspaces=true +] +> python jdma_get_monthly.py --help +usage: jdma_get_monthly.py [-h] [--start START] [--end END] [--cycle CYCLE] [--filelist] [--ens ENS] suite + +positional arguments: + suite Suite ID to extract data from + +options: + -h, --help show this help message and exit + --start START Year from which to start extracting data from + --end END Year to extract data up to (included) + --cycle CYCLE Cycle (YYYYMM) for which to extract data for. Cannot be used in combination with --start/--end + --filelist Create file(s) containing a list of files that would be extracted + --ens ENS Ensemble number of suite +(cfa_env) sci-vm-01|Tue May 26|15:27:35 BST|fix_corrupted|[priority-variables]> +\end{lstlisting} + +\clearpage + +\subsubsection{Extraction to yearly} + + +Figure \ref{fig:get-yearly} shows this graphically and Listing \ref{lst:get-yearly} gives the code implementation. + + +\begin{figure}[H] + \centering + \includegraphics[width=0.8\textwidth]{figures/get-yearly.png} + \caption{Algorithm for the \texttt{\detokenize{get-yearly.py}} file, Figure prepared using \texttt{PlantUML}. } + \label{fig:get-yearly} +\end{figure} + + + +\begin{lstlisting}[ + caption={Output of python jget-yearly.py {-}{-}help}, % Added backslash here + basicstyle=\small\ttfamily, + label={lst:get-yearly}, + frame=single, + breaklines=true, + backgroundcolor=\color{gray!5}, + columns=fullflexible, + keepspaces=true +] +> python get-yearly.py --help +usage: get-yearly.py [-h] --year YEAR --runid RUNID --member MEMBER --scenario SCENARIO --realm {ATM,OCN,CICE} + --filetype FILETYPE --var VAR + +Extract single-year variables from JDMA-retrieved data. + +options: + -h, --help show this help message and exit + --year YEAR Year to process (e.g. 1975) + --runid RUNID Run ID without the u- prefix (e.g. cz649) + --member MEMBER Ensemble member number + --scenario SCENARIO Scenario (e.g. HIST2) + --realm {ATM,OCN,CICE} + Realm + --filetype FILETYPE Type (e.g. mon__diaptr) + --var VAR Var name (e.g. zotematl) + +\end{lstlisting} + + + +\clearpage + + + +\subsection{Validation of corrupt priority-variable files} + +Figure \ref{fig:validate-files} shows the validation process graphically and Listing \ref{lst:validate} shows the code implementation. + +\begin{figure} + \centering + \includegraphics[width=0.8\textwidth]{figures/validate.png} + \caption{Algorithm for the \texttt{\detokenize{validate-files.py}} file, Figure prepared using \texttt{PlantUML}. } + \label{fig:validate-files} +\end{figure} + + + + +\begin{lstlisting}[ + caption={Output of python validate-files.py {-}{-}help}, % Added backslash here + basicstyle=\small\ttfamily, + label={lst:validate}, + frame=single, + breaklines=true, + backgroundcolor=\color{gray!5}, + columns=fullflexible, + keepspaces=true +] +> python validate-files.py --help +Usage: validate-files.py [OPTIONS] + + Scan NetCDF files and generate a full Health Report (Live Logging). + +Options: + -p, --path TEXT Path to scan. + -l, --list PATH Text file with paths. + -w, --workers INTEGER Number of parallel workers. + -o, --out TEXT Output report file. + --help Show this message and exit. +(cfa_env) sci-vm-01|Tue May 26|16:02:45 BST|fix_corrupted|[priority-variables]> +\end{lstlisting} + +The validation process is twofold; the checking of the presence of coordinate bounds, and the fidelity of the first and last slices of each diagnostic. The choice of these two tests was heuristic in the sense that there are many different tests that could be performed to assess the quality of model data but these tests were found sufficient to detect -- and subsequently fix -- all corrupt files. + +Note that to enable corrupt-file discovery the \texttt{priority-variables.py} script shown in Figure \ref{fig:priority-variables} and Listing \ref{lst:priority-variables} enables the writing of Python `glob' strings which cause the CFA generation process to fail. These lists can then be used to `home in' on specific broken files. + + + + +\clearpage + \bibliographystyle{plainnat} \bibliography{bib} -\end{document} \ No newline at end of file +\end{document} diff --git a/docs/figures/ATM.png b/docs/figures/ATM.png new file mode 100644 index 0000000..3608e65 Binary files /dev/null and b/docs/figures/ATM.png differ diff --git a/docs/figures/ATMOCNCICE.png b/docs/figures/ATMOCNCICE.png new file mode 100644 index 0000000..7b10189 Binary files /dev/null and b/docs/figures/ATMOCNCICE.png differ diff --git a/docs/figures/CICE.png b/docs/figures/CICE.png new file mode 100644 index 0000000..db579dc Binary files /dev/null and b/docs/figures/CICE.png differ diff --git a/docs/figures/OCN.png b/docs/figures/OCN.png new file mode 100644 index 0000000..e1447de Binary files /dev/null and b/docs/figures/OCN.png differ diff --git a/docs/figures/get-yearly.png b/docs/figures/get-yearly.png new file mode 100644 index 0000000..273ca86 Binary files /dev/null and b/docs/figures/get-yearly.png differ diff --git a/docs/figures/jdma_get_monthly.png b/docs/figures/jdma_get_monthly.png new file mode 100644 index 0000000..4d44b59 Binary files /dev/null and b/docs/figures/jdma_get_monthly.png differ diff --git a/docs/figures/priority-variables.png b/docs/figures/priority-variables.png new file mode 100644 index 0000000..a4f6a4e Binary files /dev/null and b/docs/figures/priority-variables.png differ diff --git a/docs/figures/seed.png b/docs/figures/seed.png index c69b019..65bb9c0 100644 Binary files a/docs/figures/seed.png and b/docs/figures/seed.png differ diff --git a/docs/figures/validate.png b/docs/figures/validate.png new file mode 100644 index 0000000..2550611 Binary files /dev/null and b/docs/figures/validate.png differ diff --git a/docs/go b/docs/go new file mode 100755 index 0000000..d758fda --- /dev/null +++ b/docs/go @@ -0,0 +1,6 @@ +# Script to automate manual PDF generation. + +pdflatex Technical_Report_CFA_CANARI +bibtex Technical_Report_CFA_CANARI +pdflatex Technical_Report_CFA_CANARI +pdflatex Technical_Report_CFA_CANARI diff --git a/docs/plantUML/get-yearly.puml b/docs/plantUML/get-yearly.puml new file mode 100644 index 0000000..d36f5d7 --- /dev/null +++ b/docs/plantUML/get-yearly.puml @@ -0,0 +1,49 @@ +@startuml +' Scale to fit A4 Portrait/Landscape +scale 0.85 +skinparam dpi 300 +skinparam shadowing false +skinparam nodesep 20 +skinparam ranksep 25 + +header (JDMA Yearly Extraction Logic: get-yearly.py) +footer PlantUML Version %version() + +start + +split + partition "Initialisation" { + :Parse CLI Arguments\n(year, runid, member, var, etc.); + :Map Realm to ID\n(ATM->a, OCN->o, CICE->i); + } + + partition "Path Construction" { + :Define **in_dir_pattern**\n(u-sid/year*Z); + :Define **file_pattern**\n(sid+cptstr_ens_filetype_*.nc); + note right: Uses wildcards for JDMA timestamps + } + +split again + partition "Data Processing (CF-Python)" { + :Read data with **cf.read**; + note right: Select by ncvar%var_name + + if (Data Found?) then (no) + #pink:Print Error; + stop + else (yes) + :Create Output Directory; + note left: exp/member/realm/yearly/year + + :Define Output Filename; + :Write Field to Disk\n(cf.write with compress=1); + endif + } +end split + +partition "Finalisation" { + :Log Performance Stats\n(Aggregate vs Write time); +} + +stop +@enduml \ No newline at end of file diff --git a/docs/plantUML/jdma_get_monthly.puml b/docs/plantUML/jdma_get_monthly.puml new file mode 100644 index 0000000..869309d --- /dev/null +++ b/docs/plantUML/jdma_get_monthly.puml @@ -0,0 +1,63 @@ +@startuml +' Scale to fit A4 +scale 0.85 +skinparam dpi 300 +skinparam shadowing false +skinparam nodesep 20 +skinparam ranksep 20 + +header CFA Metadata Processing (Two-Column Layout) +footer PlantUML Version %version() + +start + +' COLUMN 1: Setup and Parsing +split + partition "Initialisation" { + :Load Member/RunID Mapping; + :Parse CLI Args\n(Member > 1 check); + :Extract Template IDs; + } + partition "Batch Data Parsing" { + :Load member-runid.md; + if (Find 'jdma batch'?) then (yes) + :Extract Batch IDs\n& Cycle Points; + :Map Seasons to\nBatch Numbers; + else (no) + #pink:FATAL: Exit Error; + detach + endif + } + +' COLUMN 2: File Ops and Processing +split again + partition "File Preparation" { + :Identify Template File (.cfa); + if (Testing?) then (yes) + :Prepend 'testing_'; + else (no) + :Standard Name; + endif + :Copy Template to\nNew Member File; + :Open New File (r+); + } + + partition "Processing Loop" { + :Iterate NetCDF Variables; + if (Type?) then (unique_values) + :Generate Time Series; + :Apply Quarter Map; + else (uris) + :Replace RunID\n(cv575 -> New); + endif + :Update IDs from Map; + :Write to Variable; + } +end split + +partition "Finalisation" { + :Close Dataset; +} + +stop +@enduml \ No newline at end of file diff --git a/docs/plantUML/priority-variables.puml b/docs/plantUML/priority-variables.puml new file mode 100644 index 0000000..f1a24e1 --- /dev/null +++ b/docs/plantUML/priority-variables.puml @@ -0,0 +1,52 @@ +@startuml +skinparam dpi 300 +' Widen the gap between elements to prevent text overlap +skinparam nodesep 120 +skinparam ranksep 80 + +header (NEMO Metadata Fix Logic) +footer PlantUML Version %version() + +start + +partition "Initialisation" { + :Receive CLI Arguments; + note right + realm, member, + data_path, scenario + end note + :Map Member to RunID; +} + +partition "Processing Loop" { + :Iterate Unique Variables (tqdm); + :Construct search pattern\n(var_pattern); + + if (cf.read() successful?) then (yes) + :Aggregate Data\n(relaxed_identities); + else (no) + :Log to failed_patterns.txt; + stop + endif +} + +partition "Coordinate Metadata Fix" { + if (Has T-axis?) then (yes) + :Squeeze 'time_counter'\n(Redundant Dim); + if (Missing standard_name?) then (yes) + :Set 'time' as standard_name; + else (no) + :Keep Existing; + endif + else (no) + :Skip Clean-up; + endif +} + +partition "Finalisation" { + :Generate .cfa Filename; + :cf.write() to Disk; +} + +stop +@enduml \ No newline at end of file diff --git a/docs/plantUML/seed.puml b/docs/plantUML/seed.puml new file mode 100644 index 0000000..b4864b9 --- /dev/null +++ b/docs/plantUML/seed.puml @@ -0,0 +1,56 @@ +@startuml +' --- Modern Layout (No Overlap Guaranteed) --- +skinparam shadowing false +skinparam dpi 300 +skinparam nodesep 60 +skinparam ranksep 40 + +header Seed CFA File Creation Flow +footer PlantUML Version %version() + +start + +partition "Initialisation & Discovery" { + :Parse Arguments; + if (RunID Valid?) then (no) + #pink:Exit Error; + detach + else (yes) + :Build Glob Pattern; + :Search for Files; + if (Files Found?) then (no) + #pink:Print Warning & Exit; + detach + else (yes) + :cf.read(aggregate=False); + endif + endif +} + +partition "Processing" { + if (Is Realm 'ice'?) then (yes) + :Insert Missing Time Axis (T); + :Filter Out VGRDi/VGRDs; + else (no) + endif + + split + if (Atmos & Testing?) then (yes) + :Subset Specific ncvar IDs; + else (no) + :Keep All Variables; + endif + split again + :Finalise Metadata; + :Add Field Ancillary; + end split +} + +partition "Output" { + :Set Time Axis as Unlimited; + :Relativise Paths; + :Write .cfa file; +} + +stop +@enduml \ No newline at end of file diff --git a/docs/plantUML/validate.puml b/docs/plantUML/validate.puml new file mode 100644 index 0000000..2881784 --- /dev/null +++ b/docs/plantUML/validate.puml @@ -0,0 +1,68 @@ +@startuml +' Scale to fit A4 Portrait +scale 0.85 +skinparam dpi 300 +skinparam shadowing false +skinparam nodesep 20 +skinparam ranksep 25 + +header (NetCDF Validation Logic: validate_files.py) +footer PlantUML Version %version() + +start + +' COLUMN 1: File Discovery & Parallel Setup +split + partition "File Discovery" { + :Parse CLI Options\n(""--path"", ""--list"", ""--workers""); + :Identify .nc files via\nWalk and Glob; + :Remove duplicates & sort; + :Initialize Report File; + } + + partition "Parallel Execution" { + :Start ThreadPoolExecutor; + note right: Parallel workers (w) + :Submit check_file_health\nfor each file; + repeat + :Monitor tqdm progress; + :Log result to report; + note right: Live logging (buffering=1) + repeat while (More files?) is (yes) + ->no; + } + +' COLUMN 2: The Two-Stage Health Check +split again + partition "check_file_health()" { + #aliceblue:STAGE 1: Data Integrity; + note left: Uses netCDF4 + :Silence HDF5 noise; + :Test start/end slices\nof all variables; + + if (HDF5/Slicing Error?) then (yes) + -[#red]-> + :Return FAILED Stage 1; + else (no) + #aliceblue:STAGE 2: CF Metadata; + note left: Uses cf-python + :Read Field; + :Validate coordinate bounds\n(array access); + if (Metadata/Coordinate Error?) then (yes) + -[#red]-> + :Return FAILED Stage 2; + else (no) + -[#green]-> + :Return Healthy; + endif + endif + } +end split + +partition "Finalisation" { + :Print summary to console; + :Append Final Summary to file; +} + +stop +@enduml \ No newline at end of file diff --git a/priority-variables/.gitignore b/priority-variables/.gitignore new file mode 100644 index 0000000..bd000be --- /dev/null +++ b/priority-variables/.gitignore @@ -0,0 +1 @@ +final*corrupt*txt diff --git a/priority-variables/HIST2-runids b/priority-variables/HIST2-runids new file mode 100644 index 0000000..4c9e0ad --- /dev/null +++ b/priority-variables/HIST2-runids @@ -0,0 +1,3 @@ +cv575 cv625 cw345 cw356 cv827 cv976 cz547 cy436 cw342 cw343 cy375 cy376 cy537 cy811 cy866 cy873 cy877 cy879 cy880 cy881 da179 da190 da191 da192 da193 db291 db301 db303 db304 db305 cz475 cz568 cz647 cz648 cz649 dd436 dd438 dd439 dd441 dd442 + + diff --git a/priority-variables/SSP370-runids b/priority-variables/SSP370-runids new file mode 100644 index 0000000..c56b479 --- /dev/null +++ b/priority-variables/SSP370-runids @@ -0,0 +1 @@ +de814 de436 de724 de815 df220 de830 de831 de832 de850 de851 de934 de937 de938 de939 de940 df299 df300 df301 df302 df303 df933 df934 df935 df936 df937 dh412 dh413 dh415 dh416 dh417 di511 di512 di513 di514 di515 di703 di704 di705 di706 di707 diff --git a/priority-variables/batch.sl b/priority-variables/batch.sl new file mode 100755 index 0000000..7de63f2 --- /dev/null +++ b/priority-variables/batch.sl @@ -0,0 +1,33 @@ +#!/bin/bash +#SBATCH --account=canari +#SBATCH --partition=standard +#SBATCH --qos=short +##SBATCH -o %x.out +##SBATCH -e %x.err +#SBATCH -o %x_%j.out # JobName_JobID.out +#SBATCH -e %x_%j.err # JobName_JobID.err +##SBATCH --cpus-per-task=16 +#SBATCH --mem=200G +#SBATCH --time=04:00:00 + +conda activate cfa_env + +python -u priority-variables.py --member "$member" --realm "$realm" --scenario "$scenario" \ + --data_path "/gws/ssde/j25b/canari/shared/large-ensemble/priority/${scenario}/${member}/${realm}/yearly/" + +# Check the exit code of the PREVIOUS command (the python script) +if [ $? -eq 0 ]; then + echo "----------------------------------------" + echo "JOB COMPLETED SUCCESSFULLY" + echo "Finished at: $(date)" + echo "----------------------------------------" +else + echo "----------------------------------------" + echo "JOB FAILED" + echo "Exit code: $?" + echo "Finished at: $(date)" + echo "----------------------------------------" + exit 1 +fi + +# for scenario in HIST2 SSP370; do for realm in OCN CICE; do for member in {1..40}; do sbatch --export=ALL,member=$member,realm=$realm,scenario=$scenario batch.sl ; done; done; done diff --git a/priority-variables/corrupt.py b/priority-variables/corrupt.py new file mode 100644 index 0000000..82a7654 --- /dev/null +++ b/priority-variables/corrupt.py @@ -0,0 +1,79 @@ +import os +import netCDF4 as nc +from tqdm import tqdm +import concurrent.futures + +scenario = "SSP370" +realm = "CICE" +member = "16" + +base_dir = f"/gws/ssde/j25b/canari/shared/large-ensemble/priority/{scenario}/{member}/{realm}/yearly/" +output_file = f"final_corrupt_list-{scenario}-{realm}-{member}.txt" +num_workers = 4 + + +def check_integrity(file_path): + """Checks the first and last element of every variable in a file.""" + try: + with nc.Dataset(file_path, mode="r") as rootgrp: + rootgrp.set_auto_mask(False) + for var_name in rootgrp.variables: + var = rootgrp.variables[var_name] + if var.size == 0: + continue + + # Use a small slice instead of just [0] to ensure chunk de-compression + _ = var[tuple(slice(0, 1) for _ in range(var.ndim))] + + # Check the tail + if var.size > 1: + last_idx = tuple(slice(-1, None) for _ in range(var.ndim)) + _ = var[last_idx] + return None + except Exception as e: + return (file_path, str(e)) + + +# 1. Gather files +print(f"Searching for NetCDF files in {base_dir}...") +file_list = [ + os.path.join(r, f) + for r, d, fs in os.walk(base_dir) + for f in fs + if f.endswith(".nc") +] + +corrupt_files = [] + +print(f"Found {len(file_list)} files. Starting scan with {num_workers} workers...") + +# 2. Use ThreadPoolExecutor +with concurrent.futures.ThreadPoolExecutor(max_workers=num_workers) as executor: + future_to_file = {executor.submit(check_integrity, f): f for f in file_list} + + # Open the file once and append as we find errors + with open(output_file, "w") as f_out: + for future in tqdm( + concurrent.futures.as_completed(future_to_file), + total=len(file_list), + unit="file", + leave=True, + ): + res = future.result() + if res: + file_path, error_msg = res + corrupt_files.append(res) + + # Write to the local text file immediately + f_out.write(f"{file_path}\n") + f_out.flush() # Ensure it writes to disk even if script crashes later + + tqdm.write(f"\n[CORRUPT] {file_path}") + tqdm.write(f" Error: {error_msg}") + +# 3. Final Report +print(f"\nScan complete. Found {len(corrupt_files)} corrupt files.") +if corrupt_files: + print( + f"The list of corrupt files has been saved to: {os.path.abspath(output_file)}" + ) diff --git a/priority-variables/find_failed_cf_coordinate_validation-files_from_patterns.py b/priority-variables/find_failed_cf_coordinate_validation-files_from_patterns.py new file mode 100644 index 0000000..400713d --- /dev/null +++ b/priority-variables/find_failed_cf_coordinate_validation-files_from_patterns.py @@ -0,0 +1,54 @@ +import cf +import glob +import os +from tqdm import tqdm + +# Configuration +input_file = "failed_cf_coordinate_patterns.txt" +output_file = "failed_files.txt" + +# 1. Read patterns +if not os.path.exists(input_file): + print(f"Error: {input_file} not found.") + exit(1) + +with open(input_file, "r") as f: + patterns = [line.strip() for line in f if line.strip()] + +# 2. Setup output +if os.path.exists(output_file): + os.remove(output_file) + +print(f"Reading patterns from {input_file}...") + +# 3. Iterate with Progress Bars +# Outer loop for the high-level patterns +for pattern in tqdm(patterns, desc="Total Progress", unit="pattern"): + files = sorted(glob.glob(pattern)) + + if not files: + continue + + # Inner loop for the files within each pattern + # leave=False keeps the terminal clean by removing the inner bar when done + for f in tqdm(files, desc=f" Checking {pattern.split('/')[-1][:20]}...", leave=False, unit="file"): + try: + fields = cf.read(f) + if not fields: + raise ValueError("Empty file") + + field = fields[0] + + # The 'Coordinate Test' that catches the corruption + for c in field.coordinates(): + coord = field.construct(c) + if coord.has_bounds(): + # Triggers the actual read/decompression + _ = coord.get_bounds().data.array + + except Exception: + # Silent logging to the file so the progress bar isn't interrupted + with open(output_file, "a") as err_log: + err_log.write(f"{f}\n") + +print(f"\nāœ… Done. Results saved to: {output_file}") diff --git a/priority-variables/fix_corrupted/.gitignore b/priority-variables/fix_corrupted/.gitignore new file mode 100644 index 0000000..d4147d7 --- /dev/null +++ b/priority-variables/fix_corrupted/.gitignore @@ -0,0 +1,4 @@ +HIST2 +SSP370 + +u-* diff --git a/priority-variables/fix_corrupted/corrupt_files.txt b/priority-variables/fix_corrupted/corrupt_files.txt new file mode 100644 index 0000000..42daff8 --- /dev/null +++ b/priority-variables/fix_corrupted/corrupt_files.txt @@ -0,0 +1,21 @@ +/gws/ssde/j25b/canari/shared/large-ensemble/priority/HIST2/7/OCN/yearly/1976/cz547o_7_mon__diaptr_zosrfpac.nc +/gws/ssde/j25b/canari/shared/large-ensemble/priority/HIST2/8/OCN/yearly/1953/cy436o_8_mon__grid_T_botpres.nc +/gws/ssde/j25b/canari/shared/large-ensemble/priority/HIST2/7/OCN/yearly/1952/cz547o_7_mon__grid_V_vs.nc +/gws/ssde/j25b/canari/shared/large-ensemble/priority/HIST2/38/CICE/yearly/2000/dd439i_38_mon_fmelttn_ai.nc +/gws/ssde/j25b/canari/shared/large-ensemble/priority/HIST2/35/OCN/yearly/1975/cz649o_35_mon__diaptr_zotematl.nc +/gws/ssde/j25b/canari/shared/large-ensemble/priority/SSP370/10/OCN/yearly/2020/de851o_10_mon__grid_T_snow_ai_cea.nc +/gws/ssde/j25b/canari/shared/large-ensemble/priority/SSP370/16/CICE/yearly/2058/df299i_16_mon_uvel.nc +/gws/ssde/j25b/canari/shared/large-ensemble/priority/SSP370/16/OCN/yearly/2020/df299o_16_mon__diaptr_sopstldf_atl.nc +/gws/ssde/j25b/canari/shared/large-ensemble/priority/SSP370/16/OCN/yearly/2035/df299o_16_mon__diaptr_sophtadv.nc +/gws/ssde/j25b/canari/shared/large-ensemble/priority/SSP370/23/CICE/yearly/2016/df935i_23_mon_sss.nc +/gws/ssde/j25b/canari/shared/large-ensemble/priority/HIST2/9/CICE/yearly/1964/cw342i_9_day_siforceintstry.nc +/gws/ssde/j25b/canari/shared/large-ensemble/priority/SSP370/14/CICE/yearly/2039/de939i_14_mon_flatn_ai.nc +/gws/ssde/j25b/canari/shared/large-ensemble/priority/SSP370/30/OCN/yearly/2028/dh417o_30_mon__diaptr_zotemglo.nc +/gws/ssde/j25b/canari/shared/large-ensemble/priority/SSP370/24/OCN/yearly/2086/df936o_24_mon__diaptr_zomsfglo.nc +/gws/ssde/j25b/canari/shared/large-ensemble/priority/SSP370/30/OCN/yearly/2048/dh417o_30_mon__grid_T_osalttend.nc +/gws/ssde/j25b/canari/shared/large-ensemble/priority/SSP370/30/OCN/yearly/2060/dh417o_30_mon__grid_T_sohflisf.nc +/gws/ssde/j25b/canari/shared/large-ensemble/priority/SSP370/30/OCN/yearly/2075/dh417o_30_mon__grid_T_sossheig.nc +/gws/ssde/j25b/canari/shared/large-ensemble/priority/SSP370/35/CICE/yearly/2031/di515i_35_mon_apond_ai.nc +/gws/ssde/j25b/canari/shared/large-ensemble/priority/SSP370/31/CICE/yearly/2029/di511i_31_mon_strocnx.nc +/gws/ssde/j25b/canari/shared/large-ensemble/priority/SSP370/35/OCN/yearly/2098/di515o_35_mon__diaptr_zomsfpac.nc +/gws/ssde/j25b/canari/shared/large-ensemble/priority/SSP370/31/OCN/yearly/2039/di511o_31_mon__grid_T_soqlatisf.nc diff --git a/priority-variables/fix_corrupted/get-yearly.py b/priority-variables/fix_corrupted/get-yearly.py new file mode 100755 index 0000000..5152b3d --- /dev/null +++ b/priority-variables/fix_corrupted/get-yearly.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python + +import time +import cf +import os +import argparse + +# --- Configuration --- +SOURCE_DIR = os.getcwd() +TARGET_DIR = os.getcwd() + +def get_my_data(sid, ens, exp, realm, filetype, var_name, year): + """ + Extracts data for a single year from JDMA-retrieved directories. + """ + # Mapping realm names to file string identifiers + cptstr = {'ATM': 'a', 'OCN': 'o', 'CICE': 'i'}[realm] + + # JDMA retrieves into 'u-runid/YYYYMMDDT0000Z/' + # Use a wildcard to catch the specific timestamp directory + in_dir_pattern = os.path.join(SOURCE_DIR, f"u-{sid}", f"{year}*Z") + + # Construct the file pattern (e.g., cz649o_35_mon__diaptr_*.nc) + file_pattern = os.path.join(in_dir_pattern, f"{sid}{cptstr}_{ens}_{filetype}_*.nc") + + print(f'\nProcessing Year {year} | Searching: {file_pattern}') + + aggregate_time = time.time() + try: + # Select by variable name using the ncvar% identity + f = cf.read(file_pattern, + aggregate={'ncvar_identities': True, 'concatenate': False}, + select=f'ncvar%{var_name}', + dask_chunks=None) + + if not f: + print(f"āŒ No data found for variable '{var_name}' in {year}") + return + + print(f"āœ… Found {len(f)} field(s)") + + # Output directory structure: scenario/member/realm/yearly/year + out_dir = os.path.join(TARGET_DIR, exp, str(ens), realm, 'yearly', str(year)) + os.makedirs(out_dir, exist_ok=True) + + # Standardized output name + out_file_name = os.path.join(out_dir, f"{sid}{cptstr}_{ens}_{filetype}_{var_name}.nc") + print(f'Writing to: {out_file_name}') + + write_time = time.time() + cf.write(f, out_file_name, compress=1) + print(f'Done | Aggregate: {time.time()-aggregate_time:.1f}s | Write: {time.time()-write_time:.1f}s') + + except Exception as e: + print(f"šŸ’„ Failed to process year {year}: {e}") + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description='Extract single-year variables from JDMA-retrieved data.') + + parser.add_argument('--year', type=int, required=True, help='Year to process (e.g. 1975)') + parser.add_argument('--runid', required=True, help='Run ID without the u- prefix (e.g. cz649)') + parser.add_argument('--member', type=int, required=True, help='Ensemble member number') + parser.add_argument('--scenario', required=True, help='Scenario (e.g. HIST2)') + parser.add_argument('--realm', required=True, choices=['ATM', 'OCN', 'CICE'], help='Realm') + parser.add_argument('--filetype', required=True, help='Type (e.g. mon__diaptr)') + parser.add_argument('--var', required=True, help='Var name (e.g. zotematl)') + + args = parser.parse_args() + + get_my_data( + sid=args.runid, + ens=args.member, + exp=args.scenario, + realm=args.realm, + filetype=args.filetype, + var_name=args.var, + year=args.year + ) diff --git a/priority-variables/fix_corrupted/jdma_get_monthly.py b/priority-variables/fix_corrupted/jdma_get_monthly.py new file mode 100755 index 0000000..67d1764 --- /dev/null +++ b/priority-variables/fix_corrupted/jdma_get_monthly.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python +# ------------------------------------------------------------------------------------------------ +# +# Description: +# +# Usage: jdma_get.py [--filelist] [--start ] [--end ] [--cycle ] [--ens ] +# +# Options: +# --filelist - Write filepaths to file. Does not issue JDMA GET requests. +# List of files can then be passed to et_get.py +# +# --start - Year from which to start extracting data for +# +# --end - Year upto and include to extract data for +# +# --cycle - Get a single cycle (instead of start/end) +# +# --ens - Ensemble member number +# +# Example: +# jdma_get.py --filelist --start 1950 --end 1952 --ens 5 u-cv247 +# +# ------------------------------------------------------------------------------------------------ + +import os +import pwd +import sys + +from jdma_client import jdma_lib +from jdma_client import jdma_common + +# ============================================================== +# User Configurable Settings +# ============================================================== + +# Print debug statements +DEBUG = False + +# Number of months in a cycle +MONTHS_PER_CYCLE = 3 + +# If extracting files using JDMA set target location for extracted files. (ie. not using the --filelist option) +TARGET_DIR = '/work/xfc/vol10/user_cache/canari/bjharvey/temp' +TARGET_DIR = os.getcwd() + +# File types to extract +#ATMOS_FILES = ['day_','mon_','1hr_','3hr_','6hr_pt_','day_z_','mon_z_','1hr_pt_','6hr_'] +#OCEAN_FILES = ['day__grid_T','mon__grid_T','mon__grid_U','mon__grid_V','mon__diaptr'] +#CICE_FILES = ['day','mon'] + +ATMOS_FILES = [] +OCEAN_FILES = [] +OCEAN_FILES = ['mon__grid_T','mon__grid_V','mon__diaptr'] +OCEAN_FILES = ['mon__diaptr'] +CICE_FILES = [] + +# ============================================================== +# End of User Configurable Settings +# ============================================================== + +# Flags to indicate which models files are wanted from. +ATMOS = False +NEMO = False +CICE = False +if len(ATMOS_FILES) > 0: + ATMOS = True +if len(OCEAN_FILES) > 0: + NEMO = True +if len(CICE_FILES) > 0: + CICE = True + +class JDMAInterfaceError(Exception): + pass + + +def get_user_login_name(): + "get a user login name" + return pwd.getpwuid(os.getuid()).pw_name + + +class JDMAInterface(): + + def __init__(self): + username = get_user_login_name() + self.username = username + if DEBUG: + print('username: {}'.format(self.username)) + self.storage_type = 'elastictape' + self.credentials = {} + + def submit_get(self, suite, ens_num, startdate, enddate, months, flist): + """ + Submit a jdma GET job. + """ + + + FILE_LIST = flist + if FILE_LIST: + # Output list of files to extract to a file + filelist_out = '{}-{}-{}.filelist'.format(suite, startdate, enddate) + print('Writing list of files to {}\n'.format(filelist_out)) + all_files = [] + else: + # Extracting files using JDMA; set final destination for extracted files. + target_root = os.path.join(TARGET_DIR,suite) + + if not os.path.isdir(target_root): + try: + os.mkdir(target_root) + except OSError as error: + print('Failed to make directory: {} \n {}'.format(target_root, error)) + sys.exit(2) + + for year in range(startdate, enddate+1): + for month in months: + cycle = ''.join([str(year), str(month).zfill(2), '01T0000Z']) + label = os.path.join(suite, cycle) + if DEBUG: + print('label: {}'.format(label)) + + if not FILE_LIST: + # Need a separate directory for each JDMA download request. + # JDMA sets permissions on the directory to root only. + target_dir = os.path.join(target_root, cycle) + print('\nDownloading data to: {}'.format(target_dir)) + + if DEBUG: + print('Creating target directory: {}'.format(target_dir)) + + if not os.path.isdir(target_dir): + try: + os.mkdir(target_dir) + except OSError as error: + print('Failed to make directory: {} \n {}'.format(target_dir, error)) + sys.exit(2) + + # Get batch id for this cycle + batch_id = self._get_batch_id_for_path(label) + if DEBUG: + print('batch id for this cycle: {}'.format(batch_id)) + + if batch_id == None: + raise JDMAInterfaceError(('Failed to find batch id with label: {}' + ).format(label)) + + # Get path of original data + path = self._get_batch_orig_path(batch_id, label) + + # Get list of files to pull from ET + filelist = self._get_filelist(year, month, suite, ens_num) + filelist = [os.path.join(path, f) for f in filelist] + + if DEBUG: + print('filelist: {}'.format(filelist)) + + if FILE_LIST: + all_files = all_files + filelist + else: + # Issue jdma download request + resp = jdma_lib.download_files( + self.username, + batch_id=batch_id, + filelist=filelist, + target_dir=target_dir, + credentials=self.credentials) + + if DEBUG: + print('Status Code: {}'.format(resp.status_code)) + + req_id = self._resp_to_req_id(resp) + print('Request Id: {}'.format(req_id)) + + if FILE_LIST: + with open(filelist_out, 'w') as outfile: + for line in all_files: + outfile.write(line+'\n') + req_id = 0 + + return req_id + + + def _get_filelist(self, year, month, suite, ens_num): + + m = month + y = year + runid = suite.split('-')[1] + + + prefixes = [] + filelist = [] + + # Atmos Files + if ATMOS: + suite_prefix = ''.join([runid,'a']) + if ens_num: + suite_prefix = '_'.join([suite_prefix,str(ens_num)]) + + orig_times = ['day_','mon_','1hr_','3hr_','6hr_pt_'] + times = ['day_','mon_','1hr_','3hr_','6hr_pt_','day_z_','mon_z_','1hr_pt_','6hr_'] + times = ATMOS_FILES + + for time in times: + prefix = '_'.join([suite_prefix, time]) + prefixes.append(prefix) + + # Ocean Files + if NEMO: + suite_prefix = ''.join([runid,'o']) + if ens_num: + suite_prefix = '_'.join([suite_prefix,str(ens_num)]) + + #ocean_times = ['day__grid_T','mon__grid_T','mon__grid_U','mon__grid_V','mon__diaptr'] + ocean_times = OCEAN_FILES + + for time in ocean_times: + prefix = '_'.join([suite_prefix, time]) + prefixes.append(prefix) + + if ATMOS or NEMO: + if DEBUG: + print('prefixes: {}'.format(prefixes)) + + for x in range(MONTHS_PER_CYCLE): + date = ''.join([str(y),str(m).zfill(2)]) + suffix = ''.join([date,'-',date,'.nc']) + if DEBUG: + print('suffix: {}'.format(suffix)) + m = m + 1 + + for p in prefixes: + filelist.append('_'.join([p, suffix])) + + # Ice Files + if CICE: + suite_prefix = ''.join([runid,'i']) + if ens_num: + suite_prefix = '_'.join([suite_prefix,str(ens_num)]) + + #ice_times = ['day','mon'] + ice_times = CICE_FILES + + prefixes = [] + for time in ice_times: + prefix = '_'.join([suite_prefix, time]) + prefixes.append(prefix) + + m = month + for x in range(MONTHS_PER_CYCLE): + date1 = ''.join([str(y),str(m).zfill(2),'01']) + if m == 12: + m = 0 + y = y + 1 + date2 = ''.join([str(y),str(m+1).zfill(2),'01']) + suffix = ''.join([date1,'-',date2,'.nc']) + if DEBUG: + print('suffix: {}'.format(suffix)) + m = m + 1 + + for p in prefixes: + filelist.append('_'.join([p, suffix])) + + if DEBUG: + print('filenames: {}'.format(filelist)) + + return filelist + + + def _get_batch_orig_path(self, batch_id, label): + + resp = jdma_lib.get_files(self.username, batch_id=batch_id) + + try: + fields = resp.json() + except ValueError: + raise JDMAInterfaceError('unparseable response from JDMA') + + status_code = resp.status_code + if status_code != 200: + if 'error' in fields: + raise JDMAInterfaceError('JDMA request failed with HTTP status code {} and message: {}' + .format(status_code, fields['error'])) + else: + raise JDMAInterfaceError('JDMA request failed with HTTP status code {}' + .format(status_code)) + + file = fields['migrations'][0]['archives'][0]['files'][0]['path'] + orig_path = os.path.dirname(file) + + if DEBUG: + print('Original batch file path: {}'.format(orig_path)) + + return orig_path + + + def _resp_to_req_id(self, resp): + """ + returns the request ID in the response from JDMA, + or if status code was not 200, raises an exception with + the error. + """ + try: + fields = resp.json() + except ValueError: + raise JDMAInterfaceError('unparseable response from JDMA') + + status_code = resp.status_code + + if status_code == 200: + try: + return fields['request_id'] + except KeyError: + raise JDMAInterfaceError('no request ID in JDMA response') + + elif 'error' in fields: + raise JDMAInterfaceError('JDMA request failed with HTTP status code {} and message: {}' + .format(status_code, fields['error'])) + else: + raise JDMAInterfaceError('JDMA request failed with HTTP status code {}' + .format(status_code)) + + + def _get_batch_id_for_path(self, path): + id = self._get_batch_id_for_path2(path) + must_exist = 1 + if id == None and must_exist: + raise JDMAInterfaceError('could not find batch on storage for path {}'.format(path)) + else: + return id + + + def _get_batch_id_for_path2(self, path): + """ + Look up the batch with label = the supplied path + and whose location is 'ON_STORAGE' + """ + + workspace = 'canari' + + resp = jdma_lib.get_batch(self.username, + workspace=workspace, + label=path) + + if resp.status_code != 200: + if resp.status_code % 100 == 5: + sys.stderr.write(('Warning: JDMA responded with status code {} when checking for ' + 'existing batches. Assuming none found.\n' + ).format(resp.status_code)) + return None + + resp_dict = resp.json() + + if 'migrations' in resp_dict: + batches = resp_dict['migrations'] + else: + batches = [resp_dict] + + batch_ids = [batch['migration_id'] for batch in batches + if jdma_common.get_batch_stage(batch['stage']) == 'ON_STORAGE'] + + num_matches = len(batch_ids) + + if num_matches == 0: + return None + + elif num_matches == 1: + return batch_ids[0] + + else: + raise JDMAInterfaceError('found more than one batch on storage for path {} (ids={})' + .format(path, + ','.join(map(str, batch_ids)))) + + +if __name__ == '__main__': + + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument('--start', type=int, help="Year from which to start extracting data from") + parser.add_argument('--end', type=int, help="Year to extract data up to (included)") + parser.add_argument('--cycle', type=int, help="Cycle (YYYYMM) for which to extract data for. Cannot be used in combination with --start/--end") + parser.add_argument('--filelist', action="store_true", help="Create file(s) containing a list of files that would be extracted") + parser.add_argument('suite', help="Suite ID to extract data from") + parser.add_argument('--ens', type=int, help="Ensemble number of suite") + args = parser.parse_args() + + if (args.start or args.end) and args.cycle: + print('Invalid Options: --start/--end cannot be used with --cycle') + parser.print_help() + sys.exit(1) + + #suite = 'u-cv625' + #ens_num = 2 + suite = args.suite + #ens_num = args.ens_num + if args.ens: + ens_num = args.ens + else: + ens_num = None + + if args.start: + start = args.start + else: + start = 1950 + + if args.end: + end = args.end + else: + end = 2014 + + period = 'Jan {} to Dec {}'.format(start, end) + + if args.cycle: + cycle = str(args.cycle) + start = int(cycle[:4]) + end = start + months = [int(cycle[4:6])] + period = 'cycle {}'.format(cycle) + else: + months = [] + for i in range(1,13,MONTHS_PER_CYCLE): + months.append(i) + + if args.filelist: + if DEBUG: + print('Filelist true') + flist = True + action = 'Writing list of files' + else: + flist = False + action = 'Extracting files' + + print('{} for suite: {} (Ens Member {}), for {}'.format(action, suite, ens_num, period)) + + inst = JDMAInterface() + inst.submit_get(suite, ens_num, start, end, months, flist) + + + + diff --git a/priority-variables/fix_corrupted/validate-files.py b/priority-variables/fix_corrupted/validate-files.py new file mode 100644 index 0000000..59dc745 --- /dev/null +++ b/priority-variables/fix_corrupted/validate-files.py @@ -0,0 +1,127 @@ +import os +import sys +import cf +import netCDF4 as nc +import click +import concurrent.futures +import glob +from tqdm import tqdm +from datetime import datetime +from contextlib import contextmanager, redirect_stderr + +@contextmanager +def silence_stderr(): + """Context manager to suppress low-level HDF5 diagnostic noise.""" + with open(os.devnull, 'w') as fnull: + with redirect_stderr(fnull): + try: + stderr_fd = sys.stderr.fileno() + with os.fdopen(os.dup(stderr_fd), 'w') as old_stderr: + sys.stderr.flush() + os.dup2(fnull.fileno(), stderr_fd) + try: + yield + finally: + sys.stderr.flush() + os.dup2(old_stderr.fileno(), stderr_fd) + except: + yield + +def check_file_health(file_path): + """Performs two-stage integrity and metadata check.""" + # STAGE 1: Data Integrity + try: + with silence_stderr(): + with nc.Dataset(file_path, mode="r") as rootgrp: + rootgrp.set_auto_mask(False) + for var_name in rootgrp.variables: + var = rootgrp.variables[var_name] + if var.size == 0: continue + _ = var[tuple(slice(0, 1) for _ in range(var.ndim))] + if var.size > 1: + _ = var[tuple(slice(-1, None) for _ in range(var.ndim))] + except Exception: + return (file_path, False, "FAILED: Stage 1 (Data Integrity / HDF5)") + + # STAGE 2: CF Metadata + try: + field = cf.read(file_path)[0] + for c in field.coordinates(): + if field.construct(c).has_bounds(): + _ = field.construct(c).get_bounds().data.array + return (file_path, True, "Healthy") + except Exception: + return (file_path, False, "FAILED: Stage 2 (CF Metadata / Coordinates)") + +@click.command() +@click.option('--path', '-p', 'input_paths', multiple=True, help='Path to scan.') +@click.option('--list', '-l', 'file_list_path', type=click.Path(exists=True), help='Text file with paths.') +@click.option('--workers', '-w', default=1, help='Number of parallel workers.') +@click.option('--out', '-o', default='validation_report.txt', help='Output report file.') +def main(input_paths, file_list_path, workers, out): + """Scan NetCDF files and generate a full Health Report (Live Logging).""" + file_list = [] + + # 1. Gather files + for p_input in input_paths: + target = os.path.expanduser(p_input) + if os.path.isfile(target): file_list.append(target) + elif os.path.isdir(target): + for root, _, files in os.walk(target): + for f in files: + if f.endswith(".nc"): file_list.append(os.path.join(root, f)) + + if file_list_path: + with open(file_list_path, 'r') as f: + for line in f: + p = line.strip() + if p: + matches = glob.glob(p) + if matches: + file_list.extend(matches) + elif os.path.exists(p): + file_list.append(p) + + file_list = sorted(list(set(file_list))) + if not file_list: + click.secho("No files found to scan!", fg='red') + return + + click.echo(f"Found {len(file_list)} files. Scanning with {workers} worker{'s' if workers > 1 else ''}...") + + # Initialize the report file with the header + with open(out, "w") as f_out: + f_out.write(f"Validation Report - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n") + f_out.write(f"Total files to scan: {len(file_list)}\n") + f_out.write("-" * 50 + "\n") + + corrupt_count = 0 + + # 2. Parallel Processing with Live Append + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: + futures = {executor.submit(check_file_health, f): f for f in file_list} + + # buffering=1 ensures lines are written to disk as they are generated + with open(out, "a", buffering=1) as f_out: + for future in tqdm(concurrent.futures.as_completed(futures), total=len(file_list), unit="file"): + path, healthy, msg = future.result() + + if not healthy: + corrupt_count += 1 + + status = "[OK] " if healthy else "[CORRUPT]" + f_out.write(f"{status} {path} | {msg}\n") + + # 3. Final Summary + click.echo("-" * 40) + summary_text = f"Complete: {corrupt_count} corrupt / {len(file_list)} total" + click.secho(summary_text, bold=True, fg='green' if corrupt_count==0 else 'yellow') + + with open(out, "a") as f_out: + f_out.write("\n" + "-" * 50 + "\n") + f_out.write(f"Final Summary: {len(file_list) - corrupt_count} Healthy, {corrupt_count} Corrupt\n") + + click.echo(f"Full report saved to: {os.path.abspath(out)}") + +if __name__ == '__main__': + main() diff --git a/priority-variables/fix_corrupted/validation_report.txt b/priority-variables/fix_corrupted/validation_report.txt new file mode 100644 index 0000000..ac96ba7 --- /dev/null +++ b/priority-variables/fix_corrupted/validation_report.txt @@ -0,0 +1,33 @@ +Validation Report - 2026-05-26 10:35:21 +Total files to scan: 27 +-------------------------------------------------- +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/HIST2/35/OCN/yearly/1975/cz649o_35_mon__diaptr_zotematl.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/HIST2/38/CICE/yearly/2000/dd439i_38_mon_fmelttn_ai.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/HIST2/7/OCN/yearly/1952/cz547o_7_mon__grid_V_vs.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/HIST2/7/OCN/yearly/1976/cz547o_7_mon__diaptr_zosrfpac.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/HIST2/8/OCN/yearly/1953/cy436o_8_mon__grid_T_botpres.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/HIST2/9/CICE/yearly/1964/cw342i_9_day_siforceintstry.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/SSP370/10/OCN/yearly/2020/de851o_10_mon__grid_T_snow_ai_cea.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/SSP370/14/CICE/yearly/2039/de939i_14_mon_flatn_ai.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/SSP370/16/CICE/yearly/2058/df299i_16_mon_uvel.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/SSP370/16/OCN/yearly/2020/df299o_16_mon__diaptr_sopstldf_atl.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/SSP370/16/OCN/yearly/2035/df299o_16_mon__diaptr_sophtadv.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/SSP370/16/OCN/yearly/2087/df299o_16_mon__grid_V_sometauy.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/SSP370/23/CICE/yearly/2016/df935i_23_mon_sss.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/SSP370/23/CICE/yearly/2071/df935i_23_mon_shear.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/SSP370/24/OCN/yearly/2086/df936o_24_mon__diaptr_zomsfglo.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/SSP370/30/OCN/yearly/2028/dh417o_30_mon__diaptr_zotemglo.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/SSP370/30/OCN/yearly/2048/dh417o_30_mon__grid_T_osalttend.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/SSP370/30/OCN/yearly/2048/dh417o_30_mon__grid_T_pr.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/SSP370/30/OCN/yearly/2053/dh417o_30_mon__grid_T_evap_ao_cea.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/SSP370/30/OCN/yearly/2053/dh417o_30_mon__grid_T_hfrainds.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/SSP370/30/OCN/yearly/2060/dh417o_30_mon__grid_T_sohflisf.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/SSP370/30/OCN/yearly/2075/dh417o_30_mon__grid_T_sossheig.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/SSP370/31/CICE/yearly/2029/di511i_31_mon_strocnx.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/SSP370/31/OCN/yearly/2039/di511o_31_mon__grid_T_soqlatisf.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/SSP370/35/CICE/yearly/2031/di515i_35_mon_apond_ai.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/SSP370/35/OCN/yearly/2015/di515o_35_mon__diaptr_sophtldf.nc | Healthy +[OK] /gws/ssde/j25b/canari/users/jonnyhtw/canari-CF-aggregation/priority-variables/fix_corrupted/SSP370/35/OCN/yearly/2098/di515o_35_mon__diaptr_zomsfpac.nc | Healthy + +-------------------------------------------------- +Final Summary: 27 Healthy, 0 Corrupt diff --git a/priority-variables/priority-variables.pdf b/priority-variables/priority-variables.pdf new file mode 100644 index 0000000..575fe90 Binary files /dev/null and b/priority-variables/priority-variables.pdf differ diff --git a/priority-variables/priority-variables.py b/priority-variables/priority-variables.py new file mode 100644 index 0000000..ef7d8dc --- /dev/null +++ b/priority-variables/priority-variables.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python + +""" +Create seed CFA file for CANARI priority variables. +""" + +import argparse +import glob +import os +import pathlib +import re +import sys + +import cf +import click +import numpy as np +from tqdm import tqdm + +# Adds the parent directory to the search path +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +@click.command(help=__doc__) +@click.option( + "--realm", + required=True, + type=click.Choice(["ATM", "OCN", "CICE"]), + help="The climate realm", +) +@click.option( + "--verbose", + type=click.IntRange(-1, 3), + default=None, + help="Verbosity level of cf.read", +) +@click.option( + "--scenario", + required=True, + type=click.Choice(["HIST2", "SSP370"]), + help="The scenario", +) +@click.option( + "--member", + "-m", + type=int, + required=True, + help="The ensemble member number", +) +@click.option( + "--data_path", + "-d", + required=True, + type=pathlib.Path, + help="Path to the source data directory", +) +def main(realm, member, data_path, scenario, verbose): + + if scenario == "HIST2": + runids = [ + "cv575", + "cv625", + "cw345", + "cw356", + "cv827", + "cv976", + "cz547", + "cy436", + "cw342", + "cw343", + "cy375", + "cy376", + "cy537", + "cy811", + "cy866", + "cy873", + "cy877", + "cy879", + "cy880", + "cy881", + "da179", + "da190", + "da191", + "da192", + "da193", + "db291", + "db301", + "db303", + "db304", + "db305", + "cz475", + "cz568", + "cz647", + "cz648", + "cz649", + "dd436", + "dd438", + "dd439", + "dd441", + "dd442", + ] + + elif scenario == "SSP370": + runids = [ + "de814", + "de436", + "de724", + "de815", + "df220", + "de830", + "de831", + "de832", + "de850", + "de851", + "de934", + "de937", + "de938", + "de939", + "de940", + "df299", + "df300", + "df301", + "df302", + "df303", + "df933", + "df934", + "df935", + "df936", + "df937", + "dh412", + "dh413", + "dh415", + "dh416", + "dh417", + "di511", + "di512", + "di513", + "di514", + "di515", + "di703", + "di704", + "di705", + "di706", + "di707", + ] + + runid = runids[int(member) - 1] + + print( + f"scenario is {scenario}\n" + f"realm is {realm}\n" + f"member is {member}\n" + f"runid is {runid}" + ) + + # Identify discovery year + disc_year = "1950" if scenario == "HIST2" else "2015" + + realm_to_suffix = {"ATM": "a", "OCN": "o", "CICE": "i"} + + suffix = realm_to_suffix[realm] + + discovery_files = glob.glob(str(data_path / disc_year / f"{runid}{suffix}_*.nc")) + unique_vars = sorted( + { + os.path.basename(f) + .split(f"{runid}{suffix}_{member}_")[1] + .replace(".nc", "") + for f in discovery_files + } + ) + + + print(unique_vars) + print(f"The unique variables are {unique_vars}") + + print(f"Found {len(unique_vars)} variables. Starting loop...") + + for var in (pbar := tqdm(unique_vars, leave=False, ascii=True)): + # for var in (pbar := tqdm(['mon__grid_T_soqlatisf'], leave=False, ascii=True)): + pbar.set_description(f"Processing {var}") + + print(f"Now processing {var}") + + var_pattern = str(data_path / "*" / f"{runid}{suffix}_{member}_{var}.nc") + # var_pattern = str(data_path / "203[89]" / f"{runid}{suffix}_{member}_{var}.nc") + + print(f"The var_pattern is {var_pattern}") + + try: + + f = cf.read( + var_pattern, + cfa_write=["field"], + verbose=verbose, + aggregate={ + "relaxed_identities": True, + }, + ) + + for g in f: + try: + t_axis = g.dim("T") + if not t_axis.has_property("standard_name"): + t_axis.set_property("standard_name", "time") + except ValueError: + # This triggers if "T" doesn't exist; i.e. just skip to the next 'g' + continue + + filename = f"CF-1.13_seed_CANARI_{member}_{runid}_{realm}_{var}.cfa" + cf.write(f, filename, cfa={"constructs": ["field"]}, chunk_cache=256 * 2**20) + + + except Exception as e: + # Handle the error and log the failed pattern + log_file = "failed_cf_coordinate_patterns.txt" + if not os.path.exists(log_file) or var_pattern not in open(log_file).read(): + with open(log_file, "a") as f: + f.write(var_pattern + "\n") + + print(f"Error occurred with {var_pattern}. Logged to failed_patterns.txt") + + + +if __name__ == "__main__": + main()