From e724504cdcfdd733bd9c2f399334543e9f5bf7bc Mon Sep 17 00:00:00 2001 From: farhad Date: Tue, 21 Oct 2025 06:25:16 +0000 Subject: [PATCH 1/4] Refine beyond compliance check logic and update report styling --- reports/generate_reports_data.py | 40 +++++++++++++++++++++--- templates/report.html.jinja | 52 ++++++++++++++++++++++++++++++++ website/assets/scss/style.scss | 5 +++ 3 files changed, 93 insertions(+), 4 deletions(-) diff --git a/reports/generate_reports_data.py b/reports/generate_reports_data.py index d80082458..7aa95eabe 100644 --- a/reports/generate_reports_data.py +++ b/reports/generate_reports_data.py @@ -11,7 +11,7 @@ from calitp_data_analysis.sql import get_engine, query_sql # type: ignore from siuba import _, arrange, collect # type: ignore from siuba import filter as filtr # type: ignore -from siuba import mutate, pipe, rename, select, spread # type: ignore +from siuba import gather, mutate, pipe, rename, select, spread # type: ignore from siuba.sql import LazyTbl # type: ignore from tqdm import tqdm @@ -266,6 +266,7 @@ def _guideline_check(): _.reports_status, _.is_manual, _.reports_order, + _.percentage, ) >> collect() ) @@ -277,15 +278,34 @@ def generate_guideline_check(itp_id: int, publish_date, feature): >> filtr(_.organization_itp_id == itp_id) >> filtr(_.publish_date == publish_date) >> filtr(_.feature == feature) + >> filtr(~_.reports_order.isna()) # <-- filter out NAs >> select( - _.date_checked, _.check, _.reports_status, _.is_manual, _.reports_order + _.date_checked, + _.check, + _.reports_status, + _.is_manual, + _.reports_order, + _.percentage, ) >> mutate( date_checked=_.date_checked.astype(str), reports_order=_.reports_order.astype(int), check=np.where(_.is_manual, _.check + "*", _.check), ) - >> spread(_.date_checked, _.reports_status) + >> gather("metric", "value", _.reports_status, _.percentage) + >> mutate(date_stat=_.date_checked + "_" + _.metric) + >> mutate(date_stat=_.date_stat.str.replace("_reports_status$", "", regex=True)) + >> select(-_.date_checked, -_.metric) + >> spread(_.date_stat, _.value) + >> pipe( + lambda d: d + if (d.empty or d.shape[1] < 6) + else d[ + d.columns[:4].tolist() + + [d.columns[5], d.columns[4]] + + d.columns[6:].tolist() + ] + ) >> arrange(_.reports_order) >> pipe(_.fillna("")) ) @@ -455,6 +475,16 @@ def dump_report_data( with open(out_dir / "4_guideline_checks_rt.json", "w") as f: json.dump(to_rowspan_table(guideline_checks_rt, "check"), f) + # 4_guideline_checks_accessibility.json + if verbose: + print(f"Generating Accessibility guideline checks for {itp_id}") + guideline_checks_accessibility = generate_guideline_check( + itp_id, publish_date, feature="Accurate Accessibility Data" + ) + + with open(out_dir / "4_guideline_checks_accessibility.json", "w") as f: + json.dump(to_rowspan_table(guideline_checks_accessibility, "check"), f) + # 5_validation_notices.json if verbose: print(f"Generating validation codes for {itp_id}") @@ -491,7 +521,9 @@ def generate_data_by_file_path(feed_dir, pbar=None, verbose=False): if verbose: print_func = pbar.write if pbar else print print_func(f"Generating data for file: {feed_dir}") - items = feed_dir.split("/") + # Use Path.parts for cross-platform compatibility (works on Windows and Linux) + # items = feed_dir.split("/") + items = Path(feed_dir).parts year, month, itp_id = int(items[1]), items[2], items[3] dates = get_dates_year_month(year, int(month)) date_start, date_end, publish_date = dates[0], dates[1], dates[2] diff --git a/templates/report.html.jinja b/templates/report.html.jinja index dfa89a5d4..6a3e551f3 100644 --- a/templates/report.html.jinja +++ b/templates/report.html.jinja @@ -37,6 +37,12 @@ {% endmacro %} +{% macro status_class(s) -%} + {%- if s == 'PASS' -%}text-itp-green + {%- elif s == 'FAIL' -%}text-itp-red/40 + {%- endif -%} +{%- endmacro %} + {%- block title -%} Monthly GTFS Quality Reports: {{ feed_info.date_month_year }} - {{ feed_info.feed_publisher_name }} {%- endblock -%} @@ -458,6 +464,52 @@ {% endfor %} + + + + + Beyond Compliance Check + + + + + Accessibility + + + + {{ guideline_checks_accessibility["columns"][3] }} + + + {{ guideline_checks_accessibility["columns"][4] }} + + + + + {% for row in guideline_checks_accessibility["data"] %} + + + {{ row[0] }} + + + {% set s = row[3] %} + {{ show_check_status(s) }} + {% set v = row[5] %} + {% if (v is number or (v is string and v|trim != '')) and s in ['PASS','FAIL'] %} + {{ v }}% + {% endif %} + + + {% set s = row[4] %} + {{ show_check_status(s) }} + {% set v = row[6] %} + {% if (v is number or (v is string and v|trim != '')) and s in ['PASS','FAIL'] %} + {{ v }}% + {% endif %} + + + {% endfor %} + + {% if validation_codes is not defined %} diff --git a/website/assets/scss/style.scss b/website/assets/scss/style.scss index a36c927b8..0f33335aa 100644 --- a/website/assets/scss/style.scss +++ b/website/assets/scss/style.scss @@ -166,3 +166,8 @@ dialog { animation: fade-in 250ms; } } +.percentage { + margin-left: 4px; + font-size: 0.99em; + font-weight: 900; +} From d14131984d581520c2de4ab52da9a4b13aadd689 Mon Sep 17 00:00:00 2001 From: farhad Date: Tue, 4 Nov 2025 21:08:39 +0000 Subject: [PATCH 2/4] Refactor beyond_compliance_check to remove Siuba and use pandas --- reports/generate_reports_data.py | 105 +++++++++++++++++++++++++------ templates/report.html.jinja | 105 ++++++++++++++++++------------- 2 files changed, 147 insertions(+), 63 deletions(-) diff --git a/reports/generate_reports_data.py b/reports/generate_reports_data.py index 7aa95eabe..c53da609d 100644 --- a/reports/generate_reports_data.py +++ b/reports/generate_reports_data.py @@ -9,9 +9,9 @@ import pandas as pd import typer from calitp_data_analysis.sql import get_engine, query_sql # type: ignore -from siuba import _, arrange, collect # type: ignore +from siuba import _, arrange, collect # type: ignore; , spread # type: ignore from siuba import filter as filtr # type: ignore -from siuba import gather, mutate, pipe, rename, select, spread # type: ignore +from siuba import mutate, pipe, rename, select # type: ignore; , spread # type: ignore from siuba.sql import LazyTbl # type: ignore from tqdm import tqdm @@ -250,7 +250,6 @@ def generate_ave_median_vp_age(itp_id: int, date_start, date_end): ] -@cache def _guideline_check(): return ( LazyTbl( @@ -272,13 +271,67 @@ def _guideline_check(): ) +def availability_checks(df: pd.DataFrame) -> pd.DataFrame: + """ + Replaces the siuba version for accessibility checks step + using pure pandas operations. + """ + if df.empty: + return df + + # === Filter out NAs in reports_order === + df = df[~df["reports_order"].isna()].copy() + + # === Cast reports_order to int === + df["reports_order"] = df["reports_order"].astype(int) + + # === Melt (gather) === + melted = df.melt( + id_vars=["date_checked", "check", "is_manual", "reports_order"], + value_vars=["reports_status", "percentage"], + var_name="metric", + value_name="value", + ) + + # Build combined date_stat field + melted["date_stat"] = melted["date_checked"] + "_" + melted["metric"] + melted["date_stat"] = melted["date_stat"].str.replace( + "_reports_status$", "", regex=True + ) + + # Drop redundant columns + melted = melted.drop(columns=["date_checked", "metric"]) + + # Pivot (spread) + df_pivot = ( + melted.pivot_table( + index=["check", "is_manual", "reports_order"], + columns="date_stat", + values="value", + aggfunc="first", + ) + .reset_index() + .fillna("") + ) + + # Flatten columns + df_pivot.columns = [str(c) for c in df_pivot.columns] + + # Reorder columns (swap 4th and 5th) + if df_pivot.shape[1] >= 6: + cols = df_pivot.columns.tolist() + cols = cols[:4] + [cols[5], cols[4]] + cols[6:] + df_pivot = df_pivot[cols] + + return df_pivot + + def generate_guideline_check(itp_id: int, publish_date, feature): guideline_check = ( _guideline_check() >> filtr(_.organization_itp_id == itp_id) >> filtr(_.publish_date == publish_date) >> filtr(_.feature == feature) - >> filtr(~_.reports_order.isna()) # <-- filter out NAs >> select( _.date_checked, _.check, @@ -289,23 +342,9 @@ def generate_guideline_check(itp_id: int, publish_date, feature): ) >> mutate( date_checked=_.date_checked.astype(str), - reports_order=_.reports_order.astype(int), check=np.where(_.is_manual, _.check + "*", _.check), ) - >> gather("metric", "value", _.reports_status, _.percentage) - >> mutate(date_stat=_.date_checked + "_" + _.metric) - >> mutate(date_stat=_.date_stat.str.replace("_reports_status$", "", regex=True)) - >> select(-_.date_checked, -_.metric) - >> spread(_.date_stat, _.value) - >> pipe( - lambda d: d - if (d.empty or d.shape[1] < 6) - else d[ - d.columns[:4].tolist() - + [d.columns[5], d.columns[4]] - + d.columns[6:].tolist() - ] - ) + >> pipe(availability_checks) >> arrange(_.reports_order) >> pipe(_.fillna("")) ) @@ -313,6 +352,34 @@ def generate_guideline_check(itp_id: int, publish_date, feature): return guideline_check +# def generate_guideline_check(itp_id: int, publish_date, feature): +# guideline_check = ( +# _guideline_check() +# >> filtr(_.organization_itp_id == itp_id) +# >> filtr(_.publish_date == publish_date) +# >> filtr(_.feature == feature) +# >> filtr(~_.reports_order.isna()) # <-- filter out NAs +# >> select(_.date_checked, _.check, _.reports_status, _.is_manual, _.reports_order, _.percentage +# ) +# >> mutate( +# date_checked=_.date_checked.astype(str), +# reports_order = _.reports_order.astype(int), +# check=np.where(_.is_manual, _.check + "*", _.check), +# ) +# >> gather("metric", "value", _.reports_status, _.percentage) +# >> mutate(date_stat = _.date_checked + "_" + _.metric) +# >> mutate(date_stat=_.date_stat.str.replace("_reports_status$", "", regex=True)) +# >> select(-_.date_checked, -_.metric) +# >> spread(_.date_stat, _.value) +# >> pipe(lambda d: d if (d.empty or d.shape[1] < 6) +# else d[d.columns[:4].tolist() + [d.columns[5], d.columns[4]] + d.columns[6:].tolist()]) +# >> arrange(_.reports_order) +# >> pipe(_.fillna("")) +# ) + +# return guideline_check + + @cache def _validation_codes(): return ( diff --git a/templates/report.html.jinja b/templates/report.html.jinja index 6a3e551f3..f514381f8 100644 --- a/templates/report.html.jinja +++ b/templates/report.html.jinja @@ -9,7 +9,7 @@ {% if entry %} ​ {% elif showCross %} - ​ + ​ {% else %} ​ {% endif %} @@ -39,7 +39,7 @@ {% macro status_class(s) -%} {%- if s == 'PASS' -%}text-itp-green - {%- elif s == 'FAIL' -%}text-itp-red/40 + {%- elif s == 'FAIL' -%}text-itp-red/70 {%- endif -%} {%- endmacro %} @@ -391,7 +391,7 @@ The following symbols are used to denote each check's outcome on the given date:

@@ -464,51 +464,68 @@ {% endfor %} - - - - - Beyond Compliance Check - - - - - Accessibility - + {% if guideline_checks_accessibility is defined and guideline_checks_accessibility %} + + + + + Beyond Compliance Check + + + + + Accessibility + - - {{ guideline_checks_accessibility["columns"][3] }} - - - {{ guideline_checks_accessibility["columns"][4] }} - - - - - {% for row in guideline_checks_accessibility["data"] %} + + {{ guideline_checks_accessibility["columns"][3] }} + + + {{ guideline_checks_accessibility["columns"][4] }} + + + + + {% for row in guideline_checks_accessibility["data"] %} + + + {{ row[0] }} + + + {% set s = row[3] %} + {{ show_check_status(s) }} + {% set v = row[5] %} + {% if (v is number or (v is string and v|trim != '')) and s in ['PASS','FAIL'] %} + {{ v }}% + {% endif %} + + + {% set s = row[4] %} + {{ show_check_status(s) }} + {% set v = row[6] %} + {% if (v is number or (v is string and v|trim != '')) and s in ['PASS','FAIL'] %} + {{ v }}% + {% endif %} + + + {% endfor %} + + {% else %} + - - {{ row[0] }} - - - {% set s = row[3] %} - {{ show_check_status(s) }} - {% set v = row[5] %} - {% if (v is number or (v is string and v|trim != '')) and s in ['PASS','FAIL'] %} - {{ v }}% - {% endif %} - - - {% set s = row[4] %} - {{ show_check_status(s) }} - {% set v = row[6] %} - {% if (v is number or (v is string and v|trim != '')) and s in ['PASS','FAIL'] %} - {{ v }}% - {% endif %} + + Beyond Compliance Check + + + + + + + Accessibility data unavailable for this report. - {% endfor %} - + + {% endif %} From 49b31bd53e9c236589cf61ff7d710d21b53b3d22 Mon Sep 17 00:00:00 2001 From: farhad Date: Tue, 4 Nov 2025 22:26:17 +0000 Subject: [PATCH 3/4] Fixing Terraform Error: skip reports without feed_info.json to prevent KeyError in CI --- website/generate.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/website/generate.py b/website/generate.py index 7d7911608..6edee634d 100644 --- a/website/generate.py +++ b/website/generate.py @@ -247,6 +247,13 @@ def fetch_report_data(report_dir): report_data = fetch_report_data(p_report_inputs) report_data["feeds"] = entry["feeds"] + # Skip if feed_info.json is missing + if "feed_info" not in report_data: + if args.v: + print(f"Skipping {entry['report_path']} — no feed_info.json found") + pbar.update(1) + continue + # track vendors rt_vendors = report_data["feed_info"]["rt_vendors"] schedule_vendors = report_data["feed_info"]["schedule_vendors"] From 6d4ee02b0bbecb5aab3bfadca55888f77cf9e624 Mon Sep 17 00:00:00 2001 From: farhad Date: Tue, 4 Nov 2025 23:28:57 +0000 Subject: [PATCH 4/4] Fixing Terraform Error: safely handle undefined vendor lists --- templates/month.html.jinja | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/templates/month.html.jinja b/templates/month.html.jinja index d33aa1ac6..6ee9ba330 100644 --- a/templates/month.html.jinja +++ b/templates/month.html.jinja @@ -68,8 +68,10 @@
  • +{# data-schedule-vendors="{{ entry.schedule_vendors|tojson|forceescape }}" #} +{# data-rt-vendors="{{ entry.rt_vendors|tojson|forceescape }}"> #} + data-schedule-vendors="{{ (entry.schedule_vendors | default([])) | tojson | forceescape }}" + data-rt-vendors="{{ (entry.rt_vendors | default([])) | tojson | forceescape }}"> {# use flex so wrapped items will stay aligned next to icon #}