From 77ba3c2a75c582b6afb3d76de12a7b7e80332e69 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 21 Nov 2025 16:23:28 +0100 Subject: [PATCH 1/4] feat: take the whole event span into account when downsampling Signed-off-by: F.N. Claessen --- timely_beliefs/beliefs/classes.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/timely_beliefs/beliefs/classes.py b/timely_beliefs/beliefs/classes.py index 24ab716..fae5aa8 100644 --- a/timely_beliefs/beliefs/classes.py +++ b/timely_beliefs/beliefs/classes.py @@ -2217,6 +2217,25 @@ def assign_sensor_and_event_resolution(df, sensor, event_resolution): ) +def append_shifted_last_row(bdf: BeliefsDataFrame) -> BeliefsDataFrame: + """Append a new row to a BeliefsDataFrame, duplicating the last row but with its event_start set to just before its event_end.""" + # Grab the last row + last = bdf.tail(1).copy() + + # Compute the shifted event_start + level = bdf.index.names.index("event_start") + old_ts = last.index.get_level_values(level)[0] + new_ts = old_ts + bdf.event_resolution - timedelta(milliseconds=1) + + # Replace the event_start level value of the appended row + new_index = list(last.index[0]) + new_index[level] = new_ts + last.index = pd.MultiIndex.from_tuples([tuple(new_index)], names=bdf.index.names) + + # Append new row + return pd.concat([bdf, last]) + + def downsample_beliefs_data_frame( df: BeliefsDataFrame, event_resolution: timedelta, col_att_dict: dict[str, str] ) -> BeliefsDataFrame: @@ -2228,6 +2247,7 @@ def downsample_beliefs_data_frame( "belief_time" if "belief_time" in df.index.names else "belief_horizon" ) event_timing_col = "event_start" if "event_start" in df.index.names else "event_end" + df = append_shifted_last_row(df) # take full event span into account return pd.concat( [ getattr( From f670807c8441a8203d44c4bc34005bf5f6c0b6ed Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 21 Nov 2025 17:00:44 +0100 Subject: [PATCH 2/4] feat: take the whole event span into account when upsampling Signed-off-by: F.N. Claessen --- timely_beliefs/beliefs/classes.py | 22 ++-------------------- timely_beliefs/beliefs/utils.py | 28 ++++++++++++++++++++++++---- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/timely_beliefs/beliefs/classes.py b/timely_beliefs/beliefs/classes.py index fae5aa8..0a7d73d 100644 --- a/timely_beliefs/beliefs/classes.py +++ b/timely_beliefs/beliefs/classes.py @@ -1600,6 +1600,7 @@ def resample_events( df.event_resolution = event_resolution else: # upsample + df = belief_utils.append_shifted_last_row(df, event_resolution) # take full event span into account df = df.reset_index( level=[belief_timing_col, "source", "cumulative_probability"] ) @@ -2217,25 +2218,6 @@ def assign_sensor_and_event_resolution(df, sensor, event_resolution): ) -def append_shifted_last_row(bdf: BeliefsDataFrame) -> BeliefsDataFrame: - """Append a new row to a BeliefsDataFrame, duplicating the last row but with its event_start set to just before its event_end.""" - # Grab the last row - last = bdf.tail(1).copy() - - # Compute the shifted event_start - level = bdf.index.names.index("event_start") - old_ts = last.index.get_level_values(level)[0] - new_ts = old_ts + bdf.event_resolution - timedelta(milliseconds=1) - - # Replace the event_start level value of the appended row - new_index = list(last.index[0]) - new_index[level] = new_ts - last.index = pd.MultiIndex.from_tuples([tuple(new_index)], names=bdf.index.names) - - # Append new row - return pd.concat([bdf, last]) - - def downsample_beliefs_data_frame( df: BeliefsDataFrame, event_resolution: timedelta, col_att_dict: dict[str, str] ) -> BeliefsDataFrame: @@ -2247,7 +2229,7 @@ def downsample_beliefs_data_frame( "belief_time" if "belief_time" in df.index.names else "belief_horizon" ) event_timing_col = "event_start" if "event_start" in df.index.names else "event_end" - df = append_shifted_last_row(df) # take full event span into account + df = belief_utils.append_shifted_last_row(df, event_resolution) # take full event span into account return pd.concat( [ getattr( diff --git a/timely_beliefs/beliefs/utils.py b/timely_beliefs/beliefs/utils.py index 92fda27..484e6b9 100644 --- a/timely_beliefs/beliefs/utils.py +++ b/timely_beliefs/beliefs/utils.py @@ -1180,10 +1180,7 @@ def upsample_beliefs_data_frame( if isinstance(df, classes.BeliefsDataFrame): index_levels = df.index.names df = df.reset_index().set_index("event_start") - df = df.reindex(new_index) - df = df.ffill( - limit=math.ceil(resample_ratio) - 1 if resample_ratio > 1 else None, - ) + df = df.resample(event_resolution).ffill(limit=math.ceil(resample_ratio) - 1 if resample_ratio > 1 else None) df = df.dropna() if isinstance(df, classes.BeliefsDataFrame): df = df.reset_index().set_index(index_levels) @@ -1192,3 +1189,26 @@ def upsample_beliefs_data_frame( df = df.replace(unique_event_value_not_in_df, np.NaN) df.event_resolution = event_resolution return df + + +def append_shifted_last_row(bdf: "classes.BeliefsDataFrame", event_resolution: TimedeltaLike) -> "classes.BeliefsDataFrame": + """Append a new row to a BeliefsDataFrame, duplicating the last row but with its event_start set to just before its event_end.""" + + if event_resolution == timedelta(0): + return bdf + + # Grab the last row + last = bdf.tail(1).copy() + + # Compute the shifted event_start + level = bdf.index.names.index("event_start") + old_ts = last.index.get_level_values(level)[0] + new_ts = old_ts + bdf.event_resolution - timedelta(milliseconds=1) + + # Replace the event_start level value of the appended row + new_index = list(last.index[0]) + new_index[level] = new_ts + last.index = pd.MultiIndex.from_tuples([tuple(new_index)], names=bdf.index.names) + + # Append new row + return pd.concat([bdf, last]) From 86a1798254b0e7a4ae4800be2c016c3b7ce48984 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 21 Nov 2025 17:06:18 +0100 Subject: [PATCH 3/4] style: black Signed-off-by: F.N. Claessen --- timely_beliefs/beliefs/classes.py | 8 ++++++-- timely_beliefs/beliefs/utils.py | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/timely_beliefs/beliefs/classes.py b/timely_beliefs/beliefs/classes.py index 0a7d73d..6b33189 100644 --- a/timely_beliefs/beliefs/classes.py +++ b/timely_beliefs/beliefs/classes.py @@ -1600,7 +1600,9 @@ def resample_events( df.event_resolution = event_resolution else: # upsample - df = belief_utils.append_shifted_last_row(df, event_resolution) # take full event span into account + df = belief_utils.append_shifted_last_row( + df, event_resolution + ) # take full event span into account df = df.reset_index( level=[belief_timing_col, "source", "cumulative_probability"] ) @@ -2229,7 +2231,9 @@ def downsample_beliefs_data_frame( "belief_time" if "belief_time" in df.index.names else "belief_horizon" ) event_timing_col = "event_start" if "event_start" in df.index.names else "event_end" - df = belief_utils.append_shifted_last_row(df, event_resolution) # take full event span into account + df = belief_utils.append_shifted_last_row( + df, event_resolution + ) # take full event span into account return pd.concat( [ getattr( diff --git a/timely_beliefs/beliefs/utils.py b/timely_beliefs/beliefs/utils.py index 484e6b9..d8142f3 100644 --- a/timely_beliefs/beliefs/utils.py +++ b/timely_beliefs/beliefs/utils.py @@ -1180,7 +1180,9 @@ def upsample_beliefs_data_frame( if isinstance(df, classes.BeliefsDataFrame): index_levels = df.index.names df = df.reset_index().set_index("event_start") - df = df.resample(event_resolution).ffill(limit=math.ceil(resample_ratio) - 1 if resample_ratio > 1 else None) + df = df.resample(event_resolution).ffill( + limit=math.ceil(resample_ratio) - 1 if resample_ratio > 1 else None + ) df = df.dropna() if isinstance(df, classes.BeliefsDataFrame): df = df.reset_index().set_index(index_levels) @@ -1191,7 +1193,9 @@ def upsample_beliefs_data_frame( return df -def append_shifted_last_row(bdf: "classes.BeliefsDataFrame", event_resolution: TimedeltaLike) -> "classes.BeliefsDataFrame": +def append_shifted_last_row( + bdf: "classes.BeliefsDataFrame", event_resolution: TimedeltaLike +) -> "classes.BeliefsDataFrame": """Append a new row to a BeliefsDataFrame, duplicating the last row but with its event_start set to just before its event_end.""" if event_resolution == timedelta(0): From 751c5fb6fffc5b433698fadc93e8f06125d8d679 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 21 Nov 2025 17:09:50 +0100 Subject: [PATCH 4/4] docs: update inline comments style: flake8 Signed-off-by: F.N. Claessen --- timely_beliefs/beliefs/utils.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/timely_beliefs/beliefs/utils.py b/timely_beliefs/beliefs/utils.py index d8142f3..e5ec880 100644 --- a/timely_beliefs/beliefs/utils.py +++ b/timely_beliefs/beliefs/utils.py @@ -1137,25 +1137,15 @@ def upsample_beliefs_data_frame( # For example, for x = [0, 2, 0, 0, 0] => y = L1 norm + 1 = 2 + 1 = 3 unique_event_value_not_in_df = df["event_value"].abs().sum() + 1 df = df.fillna(unique_event_value_not_in_df) - if isinstance(df, classes.BeliefsDataFrame): - start = df.event_starts[0] - end = df.event_starts[-1] + from_event_resolution - else: - start = df.index[0] - end = df.index[-1] + from_event_resolution - new_index = initialize_index( - start=start, - end=end, - resolution=event_resolution, - ) - # Reindex to introduce NaN values, then forward fill by the number of steps + + # Resample to introduce NaN values, then forward fill by the number of steps # needed to have the new resolution cover the old resolution. # For example, when resampling from a resolution of 30 to 20 minutes (NB frequency is 1 hour): # event_start event_value # 2020-03-29 10:00:00+02:00 1000.0 # 2020-03-29 11:00:00+02:00 NaN # 2020-03-29 12:00:00+02:00 2000.0 - # After reindexing + # After resampling # event_start event_value # 2020-03-29 10:00:00+02:00 1000.0 # 2020-03-29 10:20:00+02:00 NaN