diff --git a/src/rra_population_model/model/train/runner.py b/src/rra_population_model/model/train/runner.py index 636ad15..b9c1b6b 100644 --- a/src/rra_population_model/model/train/runner.py +++ b/src/rra_population_model/model/train/runner.py @@ -37,13 +37,53 @@ def train_main( "log_ntl": ["log_nighttime_lights"], }[ntl_option] bd_features: list[str] = [] + + # Defining overture features. Includes both log and non-log version of the features + + overture_features = ['log_or_service_distance', + 'or_secondary_distance', + 'log_ow_coastal_inlet_distance', + 'ow_coastal_ocean_distance', + 'ow_stream_water_kde_density', + 'or_tertiary_distance', + 'ow_coastal_inlet_distance', + 'log_or_trunk_distance', + 'log_ow_stream_water_kde_density', + 'ow_stream_water_distance', + 'or_track_distance', + 'ow_waste_and_sewage_distance', + 'log_or_residential_distance', + 'or_residential_distance', + 'ow_manmade_freshwater_distance', + 'log_or_unknown_distance', + 'log_or_primary_distance', + 'log_or_motorway_distance', + 'log_ow_river_water_distance', + 'or_primary_distance', + 'log_or_unclassified_distance', + 'ow_inland_water_distance', + 'or_unknown_distance', + 'or_motorway_distance', + 'or_trunk_distance', + 'log_ow_manmade_freshwater_distance', + 'or_unclassified_distance', + 'log_ow_stream_water_distance', + 'log_ow_waste_and_sewage_distance', + 'log_ow_coastal_ocean_distance', + 'log_or_secondary_distance', + 'log_or_track_distance', + 'log_or_tertiary_distance', + 'ow_river_water_distance', + 'or_service_distance', + 'log_ow_inland_water_distance'] + model_spec = ModelSpecification( model_version=version, model_root=str(pm_data.root), output_root=str(version_root), denominator=denominator, resolution=resolution, - features=[*bd_features, *ntl_feature], + features=[*bd_features, *ntl_feature, *overture_features], ) pm_data.save_model_specification(model_spec) diff --git a/src/rra_population_model/model_prep/training_data/utils.py b/src/rra_population_model/model_prep/training_data/utils.py index a61746a..efec45f 100644 --- a/src/rra_population_model/model_prep/training_data/utils.py +++ b/src/rra_population_model/model_prep/training_data/utils.py @@ -288,6 +288,27 @@ def process_model_gdf( for measure in keep_measures: model_gdf[f"{measure}_{denominator}"] = denominator_df[measure] + # Aggregate microsoft_v8 and ghsl_r2023a features to the admin level to get the denominator of total volume for each admin first, + # then use that to weigh the overature features to the admin level. + + admin_id = model_gdf["admin_id"] + volume_weights = ["microsoft_v8_volume", "ghsl_r2023a_volume"] + weighted_features = [ + f for f in training_meta.features if f.startswith(("or_", "ow_")) + ] + for volume in volume_weights: + weight = model_gdf[f"pixel_{volume}"] * model_gdf["isection_area"] + weight_sum = weight.groupby(admin_id).transform("sum") + for feature in weighted_features: + weighted_sum = ( + (model_gdf[f"pixel_{feature}"] * weight) + .groupby(admin_id) + .transform("sum") + ) + model_gdf[f"admin_{feature}_{volume}"] = safe_divide( + weighted_sum, weight_sum + ) + for feature in training_meta.features: model_gdf[f"admin_{feature}"] = ( model_gdf[f"pixel_{feature}"] * model_gdf["admin_area_weight"]