Feature/nilesxug/overature data

src/rra_population_model/model/train/runner.py

@@ -37,13 +37,53 @@ def train_main(
         "log_ntl": ["log_nighttime_lights"],
     }[ntl_option]
     bd_features: list[str] = []
+
+    # Defining overture features. Includes both log and non-log version of the features
+
+    overture_features = ['log_or_service_distance',
+                         'or_secondary_distance',
+                         'log_ow_coastal_inlet_distance',
+                         'ow_coastal_ocean_distance',
+                         'ow_stream_water_kde_density',
+                         'or_tertiary_distance',
+                         'ow_coastal_inlet_distance',
+                         'log_or_trunk_distance',
+                         'log_ow_stream_water_kde_density',
+                         'ow_stream_water_distance',
+                         'or_track_distance',
+                         'ow_waste_and_sewage_distance',
+                         'log_or_residential_distance',
+                         'or_residential_distance',
+                         'ow_manmade_freshwater_distance',
+                         'log_or_unknown_distance',
+                         'log_or_primary_distance',
+                         'log_or_motorway_distance',
+                         'log_ow_river_water_distance',
+                         'or_primary_distance',
+                         'log_or_unclassified_distance',
+                         'ow_inland_water_distance',
+                         'or_unknown_distance',
+                         'or_motorway_distance',
+                         'or_trunk_distance',
+                         'log_ow_manmade_freshwater_distance',
+                         'or_unclassified_distance',
+                         'log_ow_stream_water_distance',
+                         'log_ow_waste_and_sewage_distance',
+                         'log_ow_coastal_ocean_distance',
+                         'log_or_secondary_distance',
+                         'log_or_track_distance',
+                         'log_or_tertiary_distance',
+                         'ow_river_water_distance',
+                         'or_service_distance',
+                         'log_ow_inland_water_distance']
+
     model_spec = ModelSpecification(
         model_version=version,
         model_root=str(pm_data.root),
         output_root=str(version_root),
         denominator=denominator,
         resolution=resolution,
-        features=[*bd_features, *ntl_feature],
+        features=[*bd_features, *ntl_feature, *overture_features],
     )
     pm_data.save_model_specification(model_spec)
 

src/rra_population_model/model_prep/training_data/utils.py

@@ -288,6 +288,27 @@ def process_model_gdf(
         for measure in keep_measures:
             model_gdf[f"{measure}_{denominator}"] = denominator_df[measure]
 
+    # Aggregate microsoft_v8 and ghsl_r2023a features to the admin level to get the denominator of total volume for each admin first,
+    # then use that to weigh the overature features to the admin level.
+
+    admin_id = model_gdf["admin_id"]
+    volume_weights = ["microsoft_v8_volume", "ghsl_r2023a_volume"]
+    weighted_features = [
+        f for f in training_meta.features if f.startswith(("or_", "ow_"))
+    ]
+    for volume in volume_weights:
+        weight = model_gdf[f"pixel_{volume}"] * model_gdf["isection_area"]
+        weight_sum = weight.groupby(admin_id).transform("sum")
+        for feature in weighted_features:
+            weighted_sum = (
+                (model_gdf[f"pixel_{feature}"] * weight)
+                .groupby(admin_id)
+                .transform("sum")
+            )
+            model_gdf[f"admin_{feature}_{volume}"] = safe_divide(
+                weighted_sum, weight_sum
+            )
+
     for feature in training_meta.features:
         model_gdf[f"admin_{feature}"] = (
             model_gdf[f"pixel_{feature}"] * model_gdf["admin_area_weight"]