diff --git a/.gitignore b/.gitignore index eb805a98ee..7b6e0dc1e9 100644 --- a/.gitignore +++ b/.gitignore @@ -153,3 +153,13 @@ cache/ # Scratch files *scratch* + +# GDX and input files +scenario_parameters.pkl +*/data/bilateralize/gdx/* +*/data/bilateralize/diagnostics/* +*/projects/*/diagnostics/* +*/data/bilateralize/*/edit_files/* +*/data/bilateralize/*/bare_files/* +*/data/bilateralize/*/GEM/* +/message_ix_models/project/newpathways/diagnostics/data diff --git a/doc/api/tools-bilateralize.rst b/doc/api/tools-bilateralize.rst new file mode 100644 index 0000000000..341343f8f0 --- /dev/null +++ b/doc/api/tools-bilateralize.rst @@ -0,0 +1,177 @@ +.. currentmodule:: message_ix_models.tools.bilateralize + +Bilateralize (:mod:`.tools.bilateralize`) +***************************************** + +.. contents:: + :local: + +Overview +======== + +This documentation will outline how to bilateralize trade flows in MESSAGEix. + +By default, trade flows are based on a “global pool” approach, wherein all +exporters export energy/commodities to a pool from which importers import based +on regionally resolved demands. + +By “bilateralizing” trade, we specify exporter and importer flows and can therefore +represent trade flows as networks. + +The bilateralization tool, **bilateralize**, is generalized for any traded commodity, +whether that is a fuel (e.g., LNG), or a material (e.g., steel). +It also explicitly represents bilateral trade “flows”, or how a fuel/commodity is +transported from exporter to importer. +These flow technologies are user defined and flexible; the most common are pipelines +(e.g., gas pipelines), maritime shipping (e.g., LNG tanker), and transmission lines. + +The ``bilateralize`` tool is a Python script that can be used to bilateralize trade flows +in MESSAGEix. It is located in the ``message-ix-models/tools/bilateralize`` directory. + +The tool follows the following steps, which are also available in ``tool/bilateralize/workflow.py``: + +Step 1 | Edit (``tools/bilateralize/prepare_edit.py``) +====================================================== + +The first step is to generate empty (or default valued) parameters that are required for +bilateralization, specified by commodity. This step requires updates to a configuration file +(``config.yaml``) that should be housed in a project directory +(e.g., ``message-ix-models/projects/newpathways-trade/config.yaml``). A template configuration +file is provided at ``message-ix-models/data/bilateralize/configs/base_config.yaml``. + +Once the configuration is updated, the user can run +``message_ix_models.tools.bilateralize.prepare_edit.generate_edit_files(log, project_name, config_name, message_regions)`` +to produce empty (or default valued) parameters as CSV files. +These CSV files will populate in ``message-ix-models/data/[your_trade_commodity]/edit_files``. + +The tools may stop if the user specifies in their config that they want to specify a trade network +(i.e., specify which regions can trade with regions). In this case, a file called ``specify_trade_network.csv`` +will appear in ``message-ix-models/data/bilateralize/[your_trade_commodity]/speciy_network_[your_trade_commodity].csv``. + +Additional functions used here include: + +- ``message_ix_models.tools.bilateralize.calculate_distance()``: + Calculates the great-circle distance between regions (TODO: update this to use explicit maritime routes) +- ``message_ix_models.tools.bilateralize.historical_calibration.build_historical_price()``: + Builds historical price dataframes +- ``message_ix_models.tools.bilateralize.mariteam_calibration.calibrate_mariteam()``: + Calibrates maritime shipping (flow technologies) using MariTEAM output. +- ``message_ix_models.tools.bilateralize.pull_gem.import_gem()``: + Imports pre-downloaded raw data from the Global Energy Monitor which is used to calibrate + the flow technology piped oil and gas + + **This step is not necessary for the following commodities + (they are already defined in ``scenario_parameters.pkl`` in + ``message-ix-models/data/bilateralize/configs/``):** + + - Biomass (``biomass_shipped``) + + - Coal (``coal_shipped``) + + - Crude Oil (``crude_shipped`` and ``crude_piped``): Note that the global pool version of + MESSAGEix names this ``oil_exp`` and ``oil_imp`` + and combines shipped and piped trade. + + - Ethanol (``eth_shipped``) + + - Fuel Oil (``foil_shipped`` and ``foil_piped``): Note that this uses the same oil pipeline + infrastructure as crude oil and light oil. + + - Light Oil (``loil_shipped`` and ``loil_piped``): See note above on pipelines + + - Liquid H2 (``lh2_shipped``) + - LNG (``LNG_shipped``) + - Methanol (``meth_shipped``) + - Piped gas (``gas_piped``) + +Step 2 | Bare (``tools/bilateralize/bare_to_scenario``) +======================================================= + +The second step is to review, edit, and transfer the files +in ``message-ix-models/data/bilateralize/[your_trade_commodity]/edit_files/``. + +The user should then review all files in ``message-ix-models/data/bilateralize/[your_trade_commodity]/edit_files/`` +and transfer these files to ``message-ix-models/data/bilateralize/[your_trade_commodity]/bare_files/``. + +By default, the following parameters are transfered automatically with default values from +``message-ix-models/data/bilateralize/[your_trade_commodity]/edit_files/`` to +``message-ix-models/data/bilateralize/[your_trade_commodity]/bare_files/``, to ensure that a scenario +can be built (even if that scenario doesn't make a ton of sense). + +- For **trade technologies** (i.e., whether or not/how much a commodity is traded): + + - ``capacity_factor`` + - ``input`` + - ``output`` + - ``technical_lifetime`` + +- For **flow technologies** (i.e., how a commodity is transported when traded, + such as via pipelines or maritime shipping) + + - ``capacity_factor`` + - ``input`` + - ``output`` + - ``relation_activity_flow`` (this links the flow technology to the trade technology) + - ``technical_lifetime`` + +The user can call ``message_ix_models.tools.bilateralize.bilateralize.build_parameter_sheets(log)`` to +pull parameters from ``message-ix-models/data/bilateralize/[your_trade_commodity]/bare_files/`` +into a dictionary of parameter dataframes that will be used to build a scenario. +Note that this function pulls from ``bare_files`` and not ``edit_files``, so the user should ensure that +the right files are transferred in the previous step. + +Additional functions here include: + +``message_ix_models.tools.bilateralize.historical_calibration.build_hist_new_capacity_flow(message_regions)``: +Builds new capacity dataframes for historical activity of flow technologies (e.g., pipelines) + +``message_ix_models.tools.bilateralize.historical_calibration.build_hist_new_capacity_trade(message_regions)``: +Builds new capacity dataframes for historical activity of trade technologies (e.g., piped gas) + +``message_ix_models.tools.bilateralize.historical_calibration.build_historical_activity(message_regions)``: +Builds historical activity dataframes + +Step 3 | Build (``tools/bilateralize/load_and_solve.py``) +========================================================= + +This step builds a scenario. + +The user should use the function +``message_ix_models.tools.bilateralize.load_and_solve.load_and_solve()``. +This will pull the base model/scenario, clone it, remove specified trade technologies, +add them back as bilateralized versions, and export to a GDX file (if specifed- the default is to not export) +and/or solve the scenario (default is to solve). Note that exporting to GDX means that it is not stored in the ixmp database. +This will also optionally solve the scenario. + +Reporting +========= +This code is in progress. + +To add a new commodity +====================== +To add a new commodity: + +- Create a new configuration file for the commodity in ``data/bilateralize/configs/`` +(see examples from existing configurations or use ``template.yaml``) + +- Add the commodity to the ``base_config.yaml`` under ``covered_trade_technologies`` + +- Run the workflow from ``prepare_edit``, then ``bare_to_scenario``, then ``load_and_solve`` + +Data +==== +All raw data required for a MESSAGEix-GLOBIOM update are currently stored in ``P:/ene_model/MESSAGE_trade``. + +Code reference +============== + +The entire module and its contents are documented recursively: + +.. currentmodule:: message_ix_models.tools + +.. autosummary:: + :toctree: _autosummary + :template: autosummary-module.rst + :recursive: + + bilateralize diff --git a/doc/api/tools.rst b/doc/api/tools.rst index 71e7fe0de2..7d88b0127f 100644 --- a/doc/api/tools.rst +++ b/doc/api/tools.rst @@ -8,6 +8,7 @@ General purpose modeling tools (:mod:`.tools`) On other pages: +- :doc:`tools-bilateralize` - :doc:`tools-costs` On this page: diff --git a/doc/index.rst b/doc/index.rst index 14043ddff6..1a6f53f0e6 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -100,6 +100,7 @@ Other submodules are documented on their respective pages: api/disutility api/report/index api/tools + api/tools-bilateralize api/tools-costs api/tools-messagev api/data-sources diff --git a/doc/whatsnew.rst b/doc/whatsnew.rst index 85e04d6ae9..40f645738d 100644 --- a/doc/whatsnew.rst +++ b/doc/whatsnew.rst @@ -6,7 +6,8 @@ Next release - :mod:`message_ix_models` supports and is tested against `Pandas 3.0.0 `_, released 2026-01-21 (:pull:`470`). - +- New module :mod:`tools.bilateralize ` + to change scenarios to a bilateral representation of trade (:pull:`438`). - Fix water module parameter bugs and refactor cooling (:pull:`405`): infrastructure M1/Mf mode fixes, regional average shares for cooling allocation, water supply level hierarchy corrections, and test suite improvements. diff --git a/message_ix_models/data/bilateralize/Gas Pipelines.xlsx b/message_ix_models/data/bilateralize/Gas Pipelines.xlsx new file mode 100644 index 0000000000..0bb2f03a1c --- /dev/null +++ b/message_ix_models/data/bilateralize/Gas Pipelines.xlsx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fa960825b010f0010ac55a66e83b3bcf2deb4bb7274fc2bb6023fc74d787baa +size 10678 diff --git a/message_ix_models/data/bilateralize/LNG Liquefaction.xlsx b/message_ix_models/data/bilateralize/LNG Liquefaction.xlsx new file mode 100644 index 0000000000..8f5707e26e --- /dev/null +++ b/message_ix_models/data/bilateralize/LNG Liquefaction.xlsx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb768d7052795e62ca8f14e34e5d255c3190cbc9ec39ca76fd55ef95ba3c5abf +size 11400 diff --git a/message_ix_models/data/bilateralize/LNG Tankers.xlsx b/message_ix_models/data/bilateralize/LNG Tankers.xlsx new file mode 100644 index 0000000000..172afc9e46 --- /dev/null +++ b/message_ix_models/data/bilateralize/LNG Tankers.xlsx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d76bb58f3fada71992f8ef33d1eb6b8ad5786785a25dba1cca10f5deb2df2729 +size 108048 diff --git a/message_ix_models/data/bilateralize/commodity_codes.yaml b/message_ix_models/data/bilateralize/commodity_codes.yaml new file mode 100644 index 0000000000..c24c4eb74e --- /dev/null +++ b/message_ix_models/data/bilateralize/commodity_codes.yaml @@ -0,0 +1,144 @@ +# UN Comtrade HS6 code to IEA conversion factors + +Biomass: + HS: + - '2703' + MESSAGE Commodity: biomass + IEA-WEB: + - PRIMSBIO + - PEAT_PRODUCTS + - PRIMARY_SOLID_BIOFUEL + - WASTE_MUNICIPAL_REN + +Coal: + HS: + - '270111' + - '270112' + - '270119' + - '2702' + - '2704' + - '2706' + MESSAGE Commodity: coal + IEA-WEB: + - ANTCOAL + - ANTHRACITE + - BITCOAL + - BKB + - BLFURGS + - BROWN + - BROWNCOAL_ND + - COALTAR + - COKCOAL + - COKEOVGS + - GASCOKE + - GASWKSGS + - HARDCOAL + - INDWASTE + - LIGNITE + - MANGAS + - MUNWASTEN + - OVENCOKE + - PATFUEL + - PEAT + - SUBCOAL + - CHARCOAL + - COAL_TAR + - COKE_OVEN_COKE_OTH + - COKING_COAL + - HARDCOAL_ND + - BLAST_FURNACE_GAS + - COKE_OVEN_GAS + - GASWORKS_GAS + - GAS_COKE + - OTH_BITCOAL + - OTH_RECOVGASES + - SUB_BITCOAL + +Crude Oil: + HS: + - '2709' + MESSAGE Commodity: crudeoil + IEA-WEB: + - CRUDE_OIL + - OIL_SHALE + +Ethanol: + HS: + - '2207' + - '3826' + MESSAGE Commodity: eth + IEA-WEB: + - BIODIESEL + - BIOGASOL + - BIOJETKERO + - OBIOLIQ + - BIOFUEL_NONSPEC + - BIOGASES + - BIOGASOLINE + - KEROSENE_JET_BIO + - LIQBIOFUEL_OTHER + +Fuel Oil: + HS: + - '2712' + - '2713' + MESSAGE Commodity: foil + IEA-WEB: + - BITUMEN + - PARWAX + - PETCOKE + - RESFUEL + - FUEL_OIL_RESIDUAL + - PARAFFIN_WAXES + - PATENT_FUEL + - PETROLEUM_COKE + +Liquid Hydrogen: + HS: + - '280410' # hydrogen + MESSAGE Commodity: lh2 + IEA-WEB: + - + +Light Oil: + HS: + - '270120' + - '2710' + - '271111' + - '271112' + - '271129' + - '290110' + - '381121' + MESSAGE Commodity: loil + IEA-WEB: + - AVGAS + - AVIATION_GASOLINE + - ETHANE + - JETGAS + - LPG + - LUBRIC + - NAPHTHA + - NONBIODIES + - NONBIOGASO + - NONBIOJETK + - ONONSPEC + - OTHKERO + - REFINGAS + - WHITESP + - GASOLINE_JET + - KEROSENE_JET_NONBIO + - KEROSENE_OTHER + - LUBRICANTS + - MOTOR_GASOLINE_NONBIO + - REFINERY_FEEDSTOCKS + - REFINERY_GAS + - WHITE_SPIRIT + - GAS_DIESEL_OIL_NONBIO + - OTH_SEC_OIL_PRODS_ND + +LNG: + HS: + - '271111' # LNG + MESSAGE Commodity: lng + IEA-WEB: + - diff --git a/message_ix_models/data/bilateralize/configs/LNG_shipped.yaml b/message_ix_models/data/bilateralize/configs/LNG_shipped.yaml new file mode 100644 index 0000000000..b041999dea --- /dev/null +++ b/message_ix_models/data/bilateralize/configs/LNG_shipped.yaml @@ -0,0 +1,55 @@ +##################### +# LNG shipped trade # +##################### +LNG_shipped_trade: + base_trade_technology: + - LNG_bal + - LNG_bunker + trade_technology: + LNG_shipped + trade_tech_suffix: + trade_tech_number: + 1 + trade_commodity: + LNG + trade_commodity_suffix: + trade_units: + GWa + trade_form: + shp + export_level: + primary + trade_level: + shipped + import_level: + secondary + flow_technologies: + - LNG_tanker_LNG + - LNG_tanker_loil + flow_tech_suffix: + flow_constraint: + global + flow_fuel_input: + LNG_tanker_LNG: + - LNG + LNG_tanker_loil: + - lightoil + flow_material_input: + flow_commodity_output: + LNG_tanker_capacity + flow_units: + Mt-km + supply_technologies: + - LNG_prod + specify_network: + False + tracked_emissions: # List of tracked emission types + bunker_technology: + LNG_tanker_LNG: + LNG_tobunker + LNG_tanker_loil: + loil_tobunker + trade_technical_lifetime: + 1 + flow_technical_lifetime: + 20 \ No newline at end of file diff --git a/message_ix_models/data/bilateralize/configs/base_config.yaml b/message_ix_models/data/bilateralize/configs/base_config.yaml new file mode 100644 index 0000000000..ac9763a123 --- /dev/null +++ b/message_ix_models/data/bilateralize/configs/base_config.yaml @@ -0,0 +1,94 @@ +# BASE CONFIGURATION FILE FOR BILATERALIZATION # + +# DATA DIRECTORIES +p_drive_location: + #/pdrive/projects/ene.model3 + P:/ene.model + +# SCENARIO INFORMATION +scenario: + start_model: + SSP_SSP2_v6.2 + start_scen: + baseline_DEFAULT + target_model: + bilat_default + target_scen: + SSP2_v6.2_bilat + first_model_year: + 2030 + regions: + R12 + +# COVERED TRADE TECHNOLOGIES +covered_trade_technologies: + - gas_piped + - LNG_shipped + - coal_shipped + - crudeoil_shipped + - crudeoil_piped + - eth_shipped + - foil_shipped + - foil_piped + - loil_shipped + - loil_piped + - biomass_shipped + - lh2_shipped + +# SHIPPING FUELS (how much of each shipping technology is propelled by each fuel) +shipping_fuels: + LNG_tanker: + LNG_tanker_LNG: 0.8 + LNG_tanker_loil: 0.2 + oil_tanker: + oil_tanker_loil: 0.9 + oil_tanker_foil: 0.07 + oil_tanker_eth: 0.03 + +# YEARS +timeframes: + year_act_list: + - 2010 + - 2015 + - 2020 + - 2025 + - 2030 + - 2035 + - 2040 + - 2045 + - 2050 + - 2055 + - 2060 + - 2070 + - 2080 + - 2090 + - 2100 + - 2110 + year_vtg_list: + - 2010 + - 2015 + - 2020 + - 2025 + - 2030 + - 2035 + - 2040 + - 2045 + - 2050 + - 2055 + - 2060 + - 2070 + - 2080 + - 2090 + - 2100 + - 2110 + +# PARAMETERS THAT NEED TO BE DEFINED +define_parameters: + base_required_par: # Parameters that need to be defined at a minimum + - input + - output + - technical_lifetime + - inv_cost + - fix_cost + - var_cost + - capacity_factor \ No newline at end of file diff --git a/message_ix_models/data/bilateralize/configs/biomass_shipped.yaml b/message_ix_models/data/bilateralize/configs/biomass_shipped.yaml new file mode 100644 index 0000000000..f91955a6a9 --- /dev/null +++ b/message_ix_models/data/bilateralize/configs/biomass_shipped.yaml @@ -0,0 +1,50 @@ +######################### +# Biomass shipped trade # +######################### +biomass_shipped_trade: + base_trade_technology: + - biomass_trd + trade_technology: + biomass_shipped + trade_tech_suffix: + trade_tech_number: + 1 + trade_commodity: + biomass + trade_commodity_suffix: + trade_units: + GWa + trade_form: + shp + export_level: + primary + trade_level: + shipped + import_level: + primary + flow_technologies: + - energy_bulk_carrier_loil + flow_tech_suffix: + flow_constraint: + global + flow_fuel_input: + energy_bulk_carrier_loil: + - lightoil + flow_material_input: + flow_commodity_output: + energy_bulk_carrier_capacity + flow_units: + Mt-km + supply_technologies: + - land_use_biomass + - biomass_imp + specify_network: + False + tracked_emissions: # List of tracked emission types + bunker_technology: + energy_bulk_carrier_loil: + loil_tobunker + trade_technical_lifetime: + 1 + flow_technical_lifetime: + 20 \ No newline at end of file diff --git a/message_ix_models/data/bilateralize/configs/coal_shipped.yaml b/message_ix_models/data/bilateralize/configs/coal_shipped.yaml new file mode 100644 index 0000000000..a604d2b5fb --- /dev/null +++ b/message_ix_models/data/bilateralize/configs/coal_shipped.yaml @@ -0,0 +1,52 @@ +###################### +# Coal shipped trade # +###################### +coal_shipped_trade: # Name of covered trade technology (e.g., oil) + base_trade_technology: + - coal_bal + - coal_trd + trade_technology: + coal_shipped + trade_tech_suffix: + trade_tech_number: + 1 + trade_commodity: + coal + trade_commodity_suffix: + trade_units: + GWa + trade_form: + shp + export_level: + primary + trade_level: + shipped + import_level: + secondary + flow_technologies: + - energy_bulk_carrier_loil + flow_tech_suffix: + flow_constraint: + global + flow_fuel_input: + energy_bulk_carrier_loil: + - lightoil + flow_material_input: + flow_commodity_output: + energy_bulk_carrier_capacity + flow_units: + Mt-km + supply_technologies: + - coal_extr + - lignite_extr + specify_network: + False + tracked_emissions: # List of tracked emission types + bunker_technology: + energy_bulk_carrier_loil: + loil_tobunker + trade_technical_lifetime: + 1 + flow_technical_lifetime: + 20 + \ No newline at end of file diff --git a/message_ix_models/data/bilateralize/configs/crudeoil_piped.yaml b/message_ix_models/data/bilateralize/configs/crudeoil_piped.yaml new file mode 100644 index 0000000000..ef33ad9126 --- /dev/null +++ b/message_ix_models/data/bilateralize/configs/crudeoil_piped.yaml @@ -0,0 +1,55 @@ +########################## +# Pipeline crude exports # +########################## +crudeoil_piped_trade: + base_trade_technology: + oil_bal + trade_technology: + crudeoil_piped + trade_tech_suffix: + trade_tech_number: + 1 + trade_commodity: + crudeoil + trade_commodity_suffix: + trade_units: + GWa + trade_form: + piped + export_level: + primary + trade_level: + piped + import_level: + secondary + flow_technologies: + - oil_pipe + flow_tech_suffix: + flow_fuel_input: + oil_pipe: + - crudeoil + flow_material_input: + oil_pipe: + - steel + flow_commodity_output: + oil_pipeline_capacity + flow_constraint: + bilateral + flow_units: + km + supply_technologies: + - oil_extr_1 + - oil_extr_2 + - oil_extr_3 + - oil_extr_4 + - oil_extr_5 + - oil_extr_6 + - oil_extr_7 + specify_network: + True + tracked_emissions: + bunker_technology: # not applicable for pipelines + trade_technical_lifetime: + 1 + flow_technical_lifetime: + 20 \ No newline at end of file diff --git a/message_ix_models/data/bilateralize/configs/crudeoil_shipped.yaml b/message_ix_models/data/bilateralize/configs/crudeoil_shipped.yaml new file mode 100644 index 0000000000..b66c5ed751 --- /dev/null +++ b/message_ix_models/data/bilateralize/configs/crudeoil_shipped.yaml @@ -0,0 +1,56 @@ +####################### +# Crude shipped trade # +########E############## +crudeoil_shipped_trade: # Name of covered trade technology (e.g., oil) + base_trade_technology: + - oil_bal + - oil_trd + trade_technology: + crudeoil_shipped + trade_tech_suffix: + trade_tech_number: + 1 + trade_commodity: + crudeoil + trade_commodity_suffix: + trade_units: + GWa + trade_form: + shp + export_level: + primary + trade_level: + shipped + import_level: + secondary + flow_technologies: + - crudeoil_tanker_loil + flow_tech_suffix: + flow_constraint: + global + flow_fuel_input: + crudeoil_tanker_loil: + - lightoil + flow_material_input: + flow_commodity_output: + crudeoil_tanker_capacity + flow_units: + Mt-km + supply_technologies: + - oil_extr_1 + - oil_extr_2 + - oil_extr_3 + - oil_extr_4 + - oil_extr_5 + - oil_extr_6 + - oil_extr_7 + specify_network: + False + tracked_emissions: # List of tracked emission types + bunker_technology: + crudeoil_tanker_loil: + loil_tobunker + trade_technical_lifetime: + 1 + flow_technical_lifetime: + 20 \ No newline at end of file diff --git a/message_ix_models/data/bilateralize/configs/eth_shipped.yaml b/message_ix_models/data/bilateralize/configs/eth_shipped.yaml new file mode 100644 index 0000000000..7ccba91a67 --- /dev/null +++ b/message_ix_models/data/bilateralize/configs/eth_shipped.yaml @@ -0,0 +1,58 @@ +######################### +# Ethanol shipped trade # +######################### +eth_shipped_trade: + base_trade_technology: + - eth_bal + - eth_bunker + trade_technology: + eth_shipped + trade_tech_suffix: + trade_tech_number: + 1 + trade_commodity: + ethanol + trade_commodity_suffix: + trade_units: + GWa + trade_form: + shp + export_level: + primary + trade_level: + shipped + import_level: + secondary + flow_technologies: + - oil_tanker_loil + - oil_tanker_eth + flow_tech_suffix: + flow_constraint: + global + flow_fuel_input: + oil_tanker_loil: + - lightoil + oil_tanker_eth: + - ethanol + flow_material_input: + flow_commodity_output: + oil_tanker_capacity + flow_units: + Mt-km + supply_technologies: + - eth_bio + - eth_bio_ccs + - liq_bio + - liq_bio_ccs + specify_network: + False + tracked_emissions: # List of tracked emission types + bunker_technology: + oil_tanker_loil: + loil_tobunker + oil_tanker_eth: + eth_tobunker + trade_technical_lifetime: + 1 + flow_technical_lifetime: + 20 \ No newline at end of file diff --git a/message_ix_models/data/bilateralize/configs/foil_piped.yaml b/message_ix_models/data/bilateralize/configs/foil_piped.yaml new file mode 100644 index 0000000000..7d80848643 --- /dev/null +++ b/message_ix_models/data/bilateralize/configs/foil_piped.yaml @@ -0,0 +1,52 @@ +############################# +# Pipeline fuel oil exports # +############################# +foil_piped_trade: + base_trade_technology: + foil_bal + trade_technology: + foil_piped + trade_tech_suffix: + trade_tech_number: + 1 + trade_commodity: + fueloil + trade_commodity_suffix: + trade_units: + GWa + trade_form: + piped + export_level: + secondary + trade_level: + piped + import_level: + secondary + flow_technologies: + - oil_pipe + flow_tech_suffix: + flow_fuel_input: + oil_pipe: + - fueloil + flow_material_input: + oil_pipe: + - steel + flow_commodity_output: + oil_pipeline_capacity + flow_constraint: + bilateral + flow_units: + km + supply_technologies: + - foil_imp + - agg_ref_MODE_vacuum_residue # Note that agg_reg has 3 modes: vacuum_residue, atm_gasoil, heavy_foil + - agg_ref_MODE_atm_gasoil + - agg_ref_MODE_heavy_foil + specify_network: + True + tracked_emissions: + bunker_technology: # not applicable for pipelines + trade_technical_lifetime: + 1 + flow_technical_lifetime: + 20 \ No newline at end of file diff --git a/message_ix_models/data/bilateralize/configs/foil_shipped.yaml b/message_ix_models/data/bilateralize/configs/foil_shipped.yaml new file mode 100644 index 0000000000..5961e39f89 --- /dev/null +++ b/message_ix_models/data/bilateralize/configs/foil_shipped.yaml @@ -0,0 +1,58 @@ +########################## +# Fuel oil shipped trade # +########################## +foil_shipped_trade: + base_trade_technology: + - foil_bal + - foil_bunker + trade_technology: + foil_shipped + trade_tech_suffix: + trade_tech_number: + 1 + trade_commodity: + fueloil + trade_commodity_suffix: + trade_units: + GWa + trade_form: + shp + export_level: + secondary + trade_level: + shipped + import_level: + secondary + flow_technologies: + - oil_tanker_loil + - oil_tanker_foil + flow_tech_suffix: + flow_constraint: + global + flow_fuel_input: + oil_tanker_loil: + - lightoil + oil_tanker_foil: + - fueloil + flow_material_input: + flow_commodity_output: + oil_tanker_capacity + flow_units: + Mt-km + supply_technologies: + - foil_imp + - agg_ref_MODE_vacuum_residue # Note that agg_reg has 3 modes: vacuum_residue, atm_gasoil, heavy_foil + - agg_ref_MODE_atm_gasoil + - agg_ref_MODE_heavy_foil + specify_network: + False + tracked_emissions: # List of tracked emission types + bunker_technology: + oil_tanker_loil: + loil_tobunker + oil_tanker_foil: + foil_tobunker + trade_technical_lifetime: + 1 + flow_technical_lifetime: + 20 \ No newline at end of file diff --git a/message_ix_models/data/bilateralize/configs/gas_piped.yaml b/message_ix_models/data/bilateralize/configs/gas_piped.yaml new file mode 100644 index 0000000000..7657e84078 --- /dev/null +++ b/message_ix_models/data/bilateralize/configs/gas_piped.yaml @@ -0,0 +1,59 @@ +######################## +# Pipeline gas exports # +######################## +gas_piped_trade: + base_trade_technology: + gas_bal + trade_technology: + gas_piped + trade_tech_suffix: + trade_tech_number: + 1 + trade_commodity: + gas + trade_commodity_suffix: + trade_units: + GWa + trade_form: + piped + export_level: + primary + trade_level: + piped + import_level: + secondary + flow_technologies: + - gas_pipe + flow_tech_suffix: + flow_fuel_input: + gas_pipe: + - gas + flow_material_input: + gas_pipe: + - steel + flow_commodity_output: + gas_pipeline_capacity + flow_constraint: + bilateral + flow_units: + km + supply_technologies: + - gas_extr_1 + - gas_extr_2 + - gas_extr_3 + - gas_extr_4 + - gas_extr_5 + - gas_extr_6 + - gas_extr_7 + specify_network: + True + tracked_emissions: + - CH4_GAS_TRANS&STORE + - TCE + - TCE_non-CO2 + - TCE_other + bunker_technology: # not applicable for pipelines + trade_technical_lifetime: + 1 + flow_technical_lifetime: + 20 \ No newline at end of file diff --git a/message_ix_models/data/bilateralize/configs/lh2_shipped.yaml b/message_ix_models/data/bilateralize/configs/lh2_shipped.yaml new file mode 100644 index 0000000000..d12775cd4d --- /dev/null +++ b/message_ix_models/data/bilateralize/configs/lh2_shipped.yaml @@ -0,0 +1,54 @@ +########################### +# Liquid H2 shipped trade # +########################### +lh2_shipped_trade: + base_trade_technology: + - lh2_trd + trade_technology: + lh2_shipped + trade_tech_suffix: + trade_tech_number: + 1 + trade_commodity: + lh2 + trade_commodity_suffix: + trade_units: + GWa + trade_form: + shp + export_level: + primary + trade_level: + shipped + import_level: + primary + flow_technologies: + - lh2_tanker_lh2 + - lh2_tanker_loil + flow_tech_suffix: + flow_constraint: + global + flow_fuel_input: + lh2_tanker_lh2: + - lh2 + lh2_tanker_loil: + - lightoil + flow_material_input: + flow_commodity_output: + lh2_tanker_capacity + flow_units: + Mt-km + supply_technologies: + - h2_liq + specify_network: + False + tracked_emissions: # List of tracked emission types + bunker_technology: + lh2_tanker_lh2: + lh2_tobunker + lh2_tanker_loil: + loil_tobunker + trade_technical_lifetime: + 1 + flow_technical_lifetime: + 20 \ No newline at end of file diff --git a/message_ix_models/data/bilateralize/configs/loil_piped.yaml b/message_ix_models/data/bilateralize/configs/loil_piped.yaml new file mode 100644 index 0000000000..a7ff95954c --- /dev/null +++ b/message_ix_models/data/bilateralize/configs/loil_piped.yaml @@ -0,0 +1,57 @@ +############################## +# Pipeline light oil exports # +############################## +loil_piped_trade: + base_trade_technology: + loil_bal + trade_technology: + loil_piped + trade_tech_suffix: + trade_tech_number: + 1 + trade_commodity: + lightoil + trade_commodity_suffix: + trade_units: + GWa + trade_form: + piped + export_level: + secondary + trade_level: + piped + import_level: + secondary + flow_technologies: + - oil_pipe + flow_tech_suffix: + flow_fuel_input: + oil_pipe: + - lightoil + flow_material_input: + oil_pipe: + - steel + flow_commodity_output: + oil_pipeline_capacity + flow_constraint: + bilateral + flow_units: + km + supply_technologies: + - loil_imp + - syn_liq + - syn_liq_ccs + - agg_ref_MODE_kerosene + - agg_ref_MODE_diesel + - agg_ref_MODE_light_foil + - agg_ref_MODE_refinery_gas + - agg_ref_MODE_gasoline + - agg_ref_MODE_refinery_gas_int + specify_network: + True + tracked_emissions: + bunker_technology: # not applicable for pipelines + trade_technical_lifetime: + 1 + flow_technical_lifetime: + 20 \ No newline at end of file diff --git a/message_ix_models/data/bilateralize/configs/loil_shipped.yaml b/message_ix_models/data/bilateralize/configs/loil_shipped.yaml new file mode 100644 index 0000000000..8f23038ba7 --- /dev/null +++ b/message_ix_models/data/bilateralize/configs/loil_shipped.yaml @@ -0,0 +1,58 @@ +########################### +# Light oil shipped trade # +########################### +loil_shipped_trade: + base_trade_technology: + - loil_bal + - loil_bunker + trade_technology: + loil_shipped + trade_tech_suffix: + trade_tech_number: + 1 + trade_commodity: + lightoil + trade_commodity_suffix: + trade_units: + GWa + trade_form: + shp + export_level: + secondary + trade_level: + shipped + import_level: + secondary + flow_technologies: + - oil_tanker_loil + flow_tech_suffix: + flow_constraint: + global + flow_fuel_input: + oil_tanker_loil: + - lightoil + flow_material_input: + flow_commodity_output: + oil_tanker_capacity + flow_units: + Mt-km + supply_technologies: + - loil_imp + - syn_liq + - syn_liq_ccs + - agg_ref_MODE_kerosene + - agg_ref_MODE_diesel + - agg_ref_MODE_light_foil + - agg_ref_MODE_refinery_gas + - agg_ref_MODE_gasoline + - agg_ref_MODE_refinery_gas_int + specify_network: + False + tracked_emissions: # List of tracked emission types + bunker_technology: + oil_tanker_loil: + loil_tobunker + trade_technical_lifetime: + 1 + flow_technical_lifetime: + 20 \ No newline at end of file diff --git a/message_ix_models/data/bilateralize/configs/template.yaml b/message_ix_models/data/bilateralize/configs/template.yaml new file mode 100644 index 0000000000..c9c508369a --- /dev/null +++ b/message_ix_models/data/bilateralize/configs/template.yaml @@ -0,0 +1,28 @@ +# TEMPLATE FOR TRADE CONFIGURATION FILES +# The trade technology is the technology that is used to transport the commodity from one node to another. +template_trade_technology: # Name of covered trade technology (e.g., oil) + base_trade_technology: # [Optional] If there is this technology in the base scenario, put here (e.g., oil_trd) + trade_technology: # Technology name used in bilateralization + trade_tech_suffix: # [Optional] Suffix on technology (e.g., UHV for elec, for ultra high voltage) + trade_tech_number: # [Optional] Number of pipes/routes per edge. + trade_commodity: # Commodity being traded (e.g., crudeoil) + trade_unit: # Units of traded commodity (e.g., GWa) + trade_commodity_suffix: # [Optional] Suffix on commodity (e.g., gei for electr in GEIDCO project) + trade_form: # How is the commodity shipped? (e.g., pipe, shp) + export_level: # Level that converts from supply to trade + trade_level: # Level at point of trade (e.g., piped, shipped) + import_level: # Level that converts from trade to use + flow_technologies: # [LIST] Technology for piping/shipping, etc. (e.g., pipe, shipping) + flow_tech_suffix: # Flow technology suffix (e.g., liquid for oil/LNG, solid for biomass/coal) + flow_constraint: # [STR] Is the flow technology constrained at the global level (i.e. maritime shipping) or bilaterally (pipelines)? + flow_fuel_input: # [DICT] What are the fuels required for flow technology (e.g., gas for gas pipelines) + flow_material_input: # What are the materials required for flow technology (e.g., steel for gas pipelines) + flow_commodity_output: # What is the flow service provided (e.g., gas_pipeline_capacity) + flow_units: # [STR] What are the units of the output commodity from flow technology? + supply_technologies: # Technologies the supply the traded commodity + # Do you want to specify the pipeline pairs? (instead of using all possible pairs) + specify_network: # If true, produces sheet of all mapped pipe technologies and regions, to be used to delete to desirable pairs. + tracked_emissions: # List of tracked emission types + bunker_technology: # [DICT] Bunker technology for the traded fuel if it exists (e.g., LNG_tobunker) + trade_technical_lifetime: # Technical lifetime of trade decision (default should be 1y) + flow_technical_lifetime: # Technical lifetime of flow technology (default is 20y) \ No newline at end of file diff --git a/message_ix_models/data/bilateralize/crudeoil_piped/specify_network_crudeoil_piped.csv b/message_ix_models/data/bilateralize/crudeoil_piped/specify_network_crudeoil_piped.csv new file mode 100644 index 0000000000..0e6d6494f9 --- /dev/null +++ b/message_ix_models/data/bilateralize/crudeoil_piped/specify_network_crudeoil_piped.csv @@ -0,0 +1,133 @@ +exporter,importer,export_technology,import_technology,"INCLUDE? (No=0, Yes=1)" +R12_WEU,R12_AFR,crudeoil_piped_exp_afr,crudeoil_piped_imp,1 +R12_WEU,R12_RCPA,crudeoil_piped_exp_rcpa,crudeoil_piped_imp,0 +R12_WEU,R12_CHN,crudeoil_piped_exp_chn,crudeoil_piped_imp,0 +R12_WEU,R12_EEU,crudeoil_piped_exp_eeu,crudeoil_piped_imp,1 +R12_WEU,R12_FSU,crudeoil_piped_exp_fsu,crudeoil_piped_imp,1 +R12_WEU,R12_LAM,crudeoil_piped_exp_lam,crudeoil_piped_imp,0 +R12_WEU,R12_MEA,crudeoil_piped_exp_mea,crudeoil_piped_imp,1 +R12_WEU,R12_NAM,crudeoil_piped_exp_nam,crudeoil_piped_imp,0 +R12_WEU,R12_PAO,crudeoil_piped_exp_pao,crudeoil_piped_imp,0 +R12_WEU,R12_PAS,crudeoil_piped_exp_pas,crudeoil_piped_imp,0 +R12_WEU,R12_SAS,crudeoil_piped_exp_sas,crudeoil_piped_imp,0 +R12_SAS,R12_AFR,crudeoil_piped_exp_afr,crudeoil_piped_imp,0 +R12_SAS,R12_RCPA,crudeoil_piped_exp_rcpa,crudeoil_piped_imp,0 +R12_SAS,R12_CHN,crudeoil_piped_exp_chn,crudeoil_piped_imp,0 +R12_SAS,R12_EEU,crudeoil_piped_exp_eeu,crudeoil_piped_imp,0 +R12_SAS,R12_FSU,crudeoil_piped_exp_fsu,crudeoil_piped_imp,1 +R12_SAS,R12_LAM,crudeoil_piped_exp_lam,crudeoil_piped_imp,0 +R12_SAS,R12_MEA,crudeoil_piped_exp_mea,crudeoil_piped_imp,1 +R12_SAS,R12_NAM,crudeoil_piped_exp_nam,crudeoil_piped_imp,0 +R12_SAS,R12_PAO,crudeoil_piped_exp_pao,crudeoil_piped_imp,0 +R12_SAS,R12_PAS,crudeoil_piped_exp_pas,crudeoil_piped_imp,1 +R12_SAS,R12_WEU,crudeoil_piped_exp_weu,crudeoil_piped_imp,0 +R12_RCPA,R12_AFR,crudeoil_piped_exp_afr,crudeoil_piped_imp,0 +R12_RCPA,R12_CHN,crudeoil_piped_exp_chn,crudeoil_piped_imp,0 +R12_RCPA,R12_EEU,crudeoil_piped_exp_eeu,crudeoil_piped_imp,0 +R12_RCPA,R12_FSU,crudeoil_piped_exp_fsu,crudeoil_piped_imp,0 +R12_RCPA,R12_LAM,crudeoil_piped_exp_lam,crudeoil_piped_imp,0 +R12_RCPA,R12_MEA,crudeoil_piped_exp_mea,crudeoil_piped_imp,0 +R12_RCPA,R12_NAM,crudeoil_piped_exp_nam,crudeoil_piped_imp,0 +R12_RCPA,R12_PAO,crudeoil_piped_exp_pao,crudeoil_piped_imp,0 +R12_RCPA,R12_PAS,crudeoil_piped_exp_pas,crudeoil_piped_imp,0 +R12_RCPA,R12_SAS,crudeoil_piped_exp_sas,crudeoil_piped_imp,0 +R12_RCPA,R12_WEU,crudeoil_piped_exp_weu,crudeoil_piped_imp,0 +R12_PAS,R12_AFR,crudeoil_piped_exp_afr,crudeoil_piped_imp,0 +R12_PAS,R12_RCPA,crudeoil_piped_exp_rcpa,crudeoil_piped_imp,1 +R12_PAS,R12_CHN,crudeoil_piped_exp_chn,crudeoil_piped_imp,1 +R12_PAS,R12_EEU,crudeoil_piped_exp_eeu,crudeoil_piped_imp,0 +R12_PAS,R12_FSU,crudeoil_piped_exp_fsu,crudeoil_piped_imp,1 +R12_PAS,R12_LAM,crudeoil_piped_exp_lam,crudeoil_piped_imp,0 +R12_PAS,R12_MEA,crudeoil_piped_exp_mea,crudeoil_piped_imp,0 +R12_PAS,R12_NAM,crudeoil_piped_exp_nam,crudeoil_piped_imp,0 +R12_PAS,R12_PAO,crudeoil_piped_exp_pao,crudeoil_piped_imp,0 +R12_PAS,R12_SAS,crudeoil_piped_exp_sas,crudeoil_piped_imp,0 +R12_PAS,R12_WEU,crudeoil_piped_exp_weu,crudeoil_piped_imp,0 +R12_PAO,R12_AFR,crudeoil_piped_exp_afr,crudeoil_piped_imp,0 +R12_PAO,R12_RCPA,crudeoil_piped_exp_rcpa,crudeoil_piped_imp,0 +R12_PAO,R12_CHN,crudeoil_piped_exp_chn,crudeoil_piped_imp,0 +R12_PAO,R12_EEU,crudeoil_piped_exp_eeu,crudeoil_piped_imp,0 +R12_PAO,R12_FSU,crudeoil_piped_exp_fsu,crudeoil_piped_imp,0 +R12_PAO,R12_LAM,crudeoil_piped_exp_lam,crudeoil_piped_imp,0 +R12_PAO,R12_MEA,crudeoil_piped_exp_mea,crudeoil_piped_imp,0 +R12_PAO,R12_NAM,crudeoil_piped_exp_nam,crudeoil_piped_imp,0 +R12_PAO,R12_PAS,crudeoil_piped_exp_pas,crudeoil_piped_imp,0 +R12_PAO,R12_SAS,crudeoil_piped_exp_sas,crudeoil_piped_imp,0 +R12_PAO,R12_WEU,crudeoil_piped_exp_weu,crudeoil_piped_imp,0 +R12_NAM,R12_AFR,crudeoil_piped_exp_afr,crudeoil_piped_imp,0 +R12_NAM,R12_RCPA,crudeoil_piped_exp_rcpa,crudeoil_piped_imp,0 +R12_NAM,R12_CHN,crudeoil_piped_exp_chn,crudeoil_piped_imp,0 +R12_NAM,R12_EEU,crudeoil_piped_exp_eeu,crudeoil_piped_imp,0 +R12_NAM,R12_FSU,crudeoil_piped_exp_fsu,crudeoil_piped_imp,0 +R12_NAM,R12_LAM,crudeoil_piped_exp_lam,crudeoil_piped_imp,1 +R12_NAM,R12_MEA,crudeoil_piped_exp_mea,crudeoil_piped_imp,0 +R12_NAM,R12_PAO,crudeoil_piped_exp_pao,crudeoil_piped_imp,0 +R12_NAM,R12_PAS,crudeoil_piped_exp_pas,crudeoil_piped_imp,0 +R12_NAM,R12_SAS,crudeoil_piped_exp_sas,crudeoil_piped_imp,0 +R12_NAM,R12_WEU,crudeoil_piped_exp_weu,crudeoil_piped_imp,0 +R12_MEA,R12_AFR,crudeoil_piped_exp_afr,crudeoil_piped_imp,1 +R12_MEA,R12_RCPA,crudeoil_piped_exp_rcpa,crudeoil_piped_imp,0 +R12_MEA,R12_CHN,crudeoil_piped_exp_chn,crudeoil_piped_imp,0 +R12_MEA,R12_EEU,crudeoil_piped_exp_eeu,crudeoil_piped_imp,1 +R12_MEA,R12_FSU,crudeoil_piped_exp_fsu,crudeoil_piped_imp,1 +R12_MEA,R12_LAM,crudeoil_piped_exp_lam,crudeoil_piped_imp,0 +R12_MEA,R12_NAM,crudeoil_piped_exp_nam,crudeoil_piped_imp,0 +R12_MEA,R12_PAO,crudeoil_piped_exp_pao,crudeoil_piped_imp,0 +R12_MEA,R12_PAS,crudeoil_piped_exp_pas,crudeoil_piped_imp,0 +R12_MEA,R12_SAS,crudeoil_piped_exp_sas,crudeoil_piped_imp,1 +R12_MEA,R12_WEU,crudeoil_piped_exp_weu,crudeoil_piped_imp,1 +R12_LAM,R12_AFR,crudeoil_piped_exp_afr,crudeoil_piped_imp,0 +R12_LAM,R12_RCPA,crudeoil_piped_exp_rcpa,crudeoil_piped_imp,0 +R12_LAM,R12_CHN,crudeoil_piped_exp_chn,crudeoil_piped_imp,0 +R12_LAM,R12_EEU,crudeoil_piped_exp_eeu,crudeoil_piped_imp,0 +R12_LAM,R12_FSU,crudeoil_piped_exp_fsu,crudeoil_piped_imp,0 +R12_LAM,R12_MEA,crudeoil_piped_exp_mea,crudeoil_piped_imp,0 +R12_LAM,R12_NAM,crudeoil_piped_exp_nam,crudeoil_piped_imp,1 +R12_LAM,R12_PAO,crudeoil_piped_exp_pao,crudeoil_piped_imp,0 +R12_LAM,R12_PAS,crudeoil_piped_exp_pas,crudeoil_piped_imp,0 +R12_LAM,R12_SAS,crudeoil_piped_exp_sas,crudeoil_piped_imp,0 +R12_LAM,R12_WEU,crudeoil_piped_exp_weu,crudeoil_piped_imp,0 +R12_FSU,R12_AFR,crudeoil_piped_exp_afr,crudeoil_piped_imp,0 +R12_FSU,R12_RCPA,crudeoil_piped_exp_rcpa,crudeoil_piped_imp,1 +R12_FSU,R12_CHN,crudeoil_piped_exp_chn,crudeoil_piped_imp,1 +R12_FSU,R12_EEU,crudeoil_piped_exp_eeu,crudeoil_piped_imp,1 +R12_FSU,R12_LAM,crudeoil_piped_exp_lam,crudeoil_piped_imp,0 +R12_FSU,R12_MEA,crudeoil_piped_exp_mea,crudeoil_piped_imp,1 +R12_FSU,R12_NAM,crudeoil_piped_exp_nam,crudeoil_piped_imp,0 +R12_FSU,R12_PAO,crudeoil_piped_exp_pao,crudeoil_piped_imp,0 +R12_FSU,R12_PAS,crudeoil_piped_exp_pas,crudeoil_piped_imp,1 +R12_FSU,R12_SAS,crudeoil_piped_exp_sas,crudeoil_piped_imp,1 +R12_FSU,R12_WEU,crudeoil_piped_exp_weu,crudeoil_piped_imp,1 +R12_EEU,R12_AFR,crudeoil_piped_exp_afr,crudeoil_piped_imp,0 +R12_EEU,R12_RCPA,crudeoil_piped_exp_rcpa,crudeoil_piped_imp,0 +R12_EEU,R12_CHN,crudeoil_piped_exp_chn,crudeoil_piped_imp,0 +R12_EEU,R12_FSU,crudeoil_piped_exp_fsu,crudeoil_piped_imp,1 +R12_EEU,R12_LAM,crudeoil_piped_exp_lam,crudeoil_piped_imp,0 +R12_EEU,R12_MEA,crudeoil_piped_exp_mea,crudeoil_piped_imp,0 +R12_EEU,R12_NAM,crudeoil_piped_exp_nam,crudeoil_piped_imp,0 +R12_EEU,R12_PAO,crudeoil_piped_exp_pao,crudeoil_piped_imp,0 +R12_EEU,R12_PAS,crudeoil_piped_exp_pas,crudeoil_piped_imp,0 +R12_EEU,R12_SAS,crudeoil_piped_exp_sas,crudeoil_piped_imp,0 +R12_EEU,R12_WEU,crudeoil_piped_exp_weu,crudeoil_piped_imp,1 +R12_CHN,R12_AFR,crudeoil_piped_exp_afr,crudeoil_piped_imp,0 +R12_CHN,R12_RCPA,crudeoil_piped_exp_rcpa,crudeoil_piped_imp,0 +R12_CHN,R12_EEU,crudeoil_piped_exp_eeu,crudeoil_piped_imp,0 +R12_CHN,R12_FSU,crudeoil_piped_exp_fsu,crudeoil_piped_imp,1 +R12_CHN,R12_LAM,crudeoil_piped_exp_lam,crudeoil_piped_imp,0 +R12_CHN,R12_MEA,crudeoil_piped_exp_mea,crudeoil_piped_imp,0 +R12_CHN,R12_NAM,crudeoil_piped_exp_nam,crudeoil_piped_imp,0 +R12_CHN,R12_PAO,crudeoil_piped_exp_pao,crudeoil_piped_imp,0 +R12_CHN,R12_PAS,crudeoil_piped_exp_pas,crudeoil_piped_imp,0 +R12_CHN,R12_SAS,crudeoil_piped_exp_sas,crudeoil_piped_imp,0 +R12_CHN,R12_WEU,crudeoil_piped_exp_weu,crudeoil_piped_imp,0 +R12_AFR,R12_RCPA,crudeoil_piped_exp_rcpa,crudeoil_piped_imp,0 +R12_AFR,R12_CHN,crudeoil_piped_exp_chn,crudeoil_piped_imp,0 +R12_AFR,R12_EEU,crudeoil_piped_exp_eeu,crudeoil_piped_imp,0 +R12_AFR,R12_FSU,crudeoil_piped_exp_fsu,crudeoil_piped_imp,0 +R12_AFR,R12_LAM,crudeoil_piped_exp_lam,crudeoil_piped_imp,0 +R12_AFR,R12_MEA,crudeoil_piped_exp_mea,crudeoil_piped_imp,1 +R12_AFR,R12_NAM,crudeoil_piped_exp_nam,crudeoil_piped_imp,0 +R12_AFR,R12_PAO,crudeoil_piped_exp_pao,crudeoil_piped_imp,0 +R12_AFR,R12_PAS,crudeoil_piped_exp_pas,crudeoil_piped_imp,0 +R12_AFR,R12_SAS,crudeoil_piped_exp_sas,crudeoil_piped_imp,0 +R12_AFR,R12_WEU,crudeoil_piped_exp_weu,crudeoil_piped_imp,1 diff --git a/message_ix_models/data/bilateralize/distances/R12_distances.csv b/message_ix_models/data/bilateralize/distances/R12_distances.csv new file mode 100644 index 0000000000..35a3a7b196 --- /dev/null +++ b/message_ix_models/data/bilateralize/distances/R12_distances.csv @@ -0,0 +1,133 @@ +Node1,Port1,Node2,Port2,Distance_km +R12_AFR,Port of Durban (South Africa),R12_RCPA,Danang (Vietnam),10933.74 +R12_AFR,Port of Durban (South Africa),R12_CHN,Shanghai (China),13062.75 +R12_AFR,Port of Durban (South Africa),R12_EEU,Swinoujscie (Poland),14407.37 +R12_AFR,Port of Durban (South Africa),R12_FSU,Arctic LNG 2 (TBC) (Russia),17780.4 +R12_AFR,Port of Durban (South Africa),R12_LAM,DTSE/GEGUA Oil Terminal (Brazil),7722.88 +R12_AFR,Port of Durban (South Africa),R12_MEA,Jeddah (Saudi Arabia),7394.28 +R12_AFR,Port of Durban (South Africa),R12_NAM,Houston (United States),15541.35 +R12_AFR,Port of Durban (South Africa),R12_PAO,Hakodate (Japan),15074.5 +R12_AFR,Port of Durban (South Africa),R12_PAS,Jakarta (Indonesia),8460.8 +R12_AFR,Port of Durban (South Africa),R12_SAS,Jawahar Dweep (India),7241.41 +R12_AFR,Port of Durban (South Africa),R12_WEU,Hamburg (Germany),13615.49 +R12_RCPA,Danang (Vietnam),R12_CHN,Shanghai (China),2519.71 +R12_RCPA,Danang (Vietnam),R12_EEU,Swinoujscie (Poland),17766.42 +R12_RCPA,Danang (Vietnam),R12_FSU,Arctic LNG 2 (TBC) (Russia),13489.31 +R12_RCPA,Danang (Vietnam),R12_LAM,DTSE/GEGUA Oil Terminal (Brazil),18234.9 +R12_RCPA,Danang (Vietnam),R12_MEA,Jeddah (Saudi Arabia),10100.25 +R12_RCPA,Danang (Vietnam),R12_NAM,Houston (United States),21229.27 +R12_RCPA,Danang (Vietnam),R12_PAO,Hakodate (Japan),4531.46 +R12_RCPA,Danang (Vietnam),R12_PAS,Jakarta (Indonesia),2654.2 +R12_RCPA,Danang (Vietnam),R12_SAS,Jawahar Dweep (India),6683.01 +R12_RCPA,Danang (Vietnam),R12_WEU,Hamburg (Germany),17656.34 +R12_CHN,Shanghai (China),R12_EEU,Swinoujscie (Poland),15461.62 +R12_CHN,Shanghai (China),R12_FSU,Arctic LNG 2 (TBC) (Russia),11184.51 +R12_CHN,Shanghai (China),R12_LAM,DTSE/GEGUA Oil Terminal (Brazil),20363.9 +R12_CHN,Shanghai (China),R12_MEA,Jeddah (Saudi Arabia),12234.57 +R12_CHN,Shanghai (China),R12_NAM,Houston (United States),18924.48 +R12_CHN,Shanghai (China),R12_PAO,Hakodate (Japan),2226.67 +R12_CHN,Shanghai (China),R12_PAS,Jakarta (Indonesia),4783.2 +R12_CHN,Shanghai (China),R12_SAS,Jawahar Dweep (India),8817.33 +R12_CHN,Shanghai (China),R12_WEU,Hamburg (Germany),15351.54 +R12_EEU,Swinoujscie (Poland),R12_FSU,Arctic LNG 2 (TBC) (Russia),5022.3 +R12_EEU,Swinoujscie (Poland),R12_LAM,DTSE/GEGUA Oil Terminal (Brazil),11121.92 +R12_EEU,Swinoujscie (Poland),R12_MEA,Jeddah (Saudi Arabia),8852.73 +R12_EEU,Swinoujscie (Poland),R12_NAM,Houston (United States),10043.47 +R12_EEU,Swinoujscie (Poland),R12_PAO,Hakodate (Japan),13383.91 +R12_EEU,Swinoujscie (Poland),R12_PAS,Jakarta (Indonesia),17366.83 +R12_EEU,Swinoujscie (Poland),R12_SAS,Jawahar Dweep (India),13074.08 +R12_EEU,Swinoujscie (Poland),R12_WEU,Hamburg (Germany),1193.9 +R12_FSU,Arctic LNG 2 (TBC) (Russia),R12_LAM,DTSE/GEGUA Oil Terminal (Brazil),14482.85 +R12_FSU,Arctic LNG 2 (TBC) (Russia),R12_MEA,Jeddah (Saudi Arabia),12247.89 +R12_FSU,Arctic LNG 2 (TBC) (Russia),R12_NAM,Houston (United States),12417.93 +R12_FSU,Arctic LNG 2 (TBC) (Russia),R12_PAO,Hakodate (Japan),9106.8 +R12_FSU,Arctic LNG 2 (TBC) (Russia),R12_PAS,Jakarta (Indonesia),15752.79 +R12_FSU,Arctic LNG 2 (TBC) (Russia),R12_SAS,Jawahar Dweep (India),16469.24 +R12_FSU,Arctic LNG 2 (TBC) (Russia),R12_WEU,Hamburg (Germany),4912.22 +R12_LAM,DTSE/GEGUA Oil Terminal (Brazil),R12_MEA,Jeddah (Saudi Arabia),12965.1 +R12_LAM,DTSE/GEGUA Oil Terminal (Brazil),R12_NAM,Houston (United States),10138.75 +R12_LAM,DTSE/GEGUA Oil Terminal (Brazil),R12_PAO,Hakodate (Japan),22158.64 +R12_LAM,DTSE/GEGUA Oil Terminal (Brazil),R12_PAS,Jakarta (Indonesia),15761.96 +R12_LAM,DTSE/GEGUA Oil Terminal (Brazil),R12_SAS,Jawahar Dweep (India),14773.15 +R12_LAM,DTSE/GEGUA Oil Terminal (Brazil),R12_WEU,Hamburg (Germany),10330.03 +R12_MEA,Jeddah (Saudi Arabia),R12_NAM,Houston (United States),13833.28 +R12_MEA,Jeddah (Saudi Arabia),R12_PAO,Hakodate (Japan),14246.32 +R12_MEA,Jeddah (Saudi Arabia),R12_PAS,Jakarta (Indonesia),8683.39 +R12_MEA,Jeddah (Saudi Arabia),R12_SAS,Jawahar Dweep (India),4390.63 +R12_MEA,Jeddah (Saudi Arabia),R12_WEU,Hamburg (Germany),8060.85 +R12_NAM,Houston (United States),R12_PAO,Hakodate (Japan),16793.43 +R12_NAM,Houston (United States),R12_PAS,Jakarta (Indonesia),22347.38 +R12_NAM,Houston (United States),R12_SAS,Jawahar Dweep (India),18054.62 +R12_NAM,Houston (United States),R12_WEU,Hamburg (Germany),9724.35 +R12_PAO,Hakodate (Japan),R12_PAS,Jakarta (Indonesia),6794.95 +R12_PAO,Hakodate (Japan),R12_SAS,Jawahar Dweep (India),10829.08 +R12_PAO,Hakodate (Japan),R12_WEU,Hamburg (Germany),13273.83 +R12_PAS,Jakarta (Indonesia),R12_SAS,Jawahar Dweep (India),5333.67 +R12_PAS,Jakarta (Indonesia),R12_WEU,Hamburg (Germany),16574.95 +R12_SAS,Jawahar Dweep (India),R12_WEU,Hamburg (Germany),12282.19 +R12_RCPA,Danang (Vietnam),R12_AFR,Port of Durban (South Africa),10933.74 +R12_CHN,Shanghai (China),R12_AFR,Port of Durban (South Africa),13062.75 +R12_EEU,Swinoujscie (Poland),R12_AFR,Port of Durban (South Africa),14407.37 +R12_FSU,Arctic LNG 2 (TBC) (Russia),R12_AFR,Port of Durban (South Africa),17780.4 +R12_LAM,DTSE/GEGUA Oil Terminal (Brazil),R12_AFR,Port of Durban (South Africa),7722.88 +R12_MEA,Jeddah (Saudi Arabia),R12_AFR,Port of Durban (South Africa),7394.28 +R12_NAM,Houston (United States),R12_AFR,Port of Durban (South Africa),15541.35 +R12_PAO,Hakodate (Japan),R12_AFR,Port of Durban (South Africa),15074.5 +R12_PAS,Jakarta (Indonesia),R12_AFR,Port of Durban (South Africa),8460.8 +R12_SAS,Jawahar Dweep (India),R12_AFR,Port of Durban (South Africa),7241.41 +R12_WEU,Hamburg (Germany),R12_AFR,Port of Durban (South Africa),13615.49 +R12_CHN,Shanghai (China),R12_RCPA,Danang (Vietnam),2519.71 +R12_EEU,Swinoujscie (Poland),R12_RCPA,Danang (Vietnam),17766.42 +R12_FSU,Arctic LNG 2 (TBC) (Russia),R12_RCPA,Danang (Vietnam),13489.31 +R12_LAM,DTSE/GEGUA Oil Terminal (Brazil),R12_RCPA,Danang (Vietnam),18234.9 +R12_MEA,Jeddah (Saudi Arabia),R12_RCPA,Danang (Vietnam),10100.25 +R12_NAM,Houston (United States),R12_RCPA,Danang (Vietnam),21229.27 +R12_PAO,Hakodate (Japan),R12_RCPA,Danang (Vietnam),4531.46 +R12_PAS,Jakarta (Indonesia),R12_RCPA,Danang (Vietnam),2654.2 +R12_SAS,Jawahar Dweep (India),R12_RCPA,Danang (Vietnam),6683.01 +R12_WEU,Hamburg (Germany),R12_RCPA,Danang (Vietnam),17656.34 +R12_EEU,Swinoujscie (Poland),R12_CHN,Shanghai (China),15461.62 +R12_FSU,Arctic LNG 2 (TBC) (Russia),R12_CHN,Shanghai (China),11184.51 +R12_LAM,DTSE/GEGUA Oil Terminal (Brazil),R12_CHN,Shanghai (China),20363.9 +R12_MEA,Jeddah (Saudi Arabia),R12_CHN,Shanghai (China),12234.57 +R12_NAM,Houston (United States),R12_CHN,Shanghai (China),18924.48 +R12_PAO,Hakodate (Japan),R12_CHN,Shanghai (China),2226.67 +R12_PAS,Jakarta (Indonesia),R12_CHN,Shanghai (China),4783.2 +R12_SAS,Jawahar Dweep (India),R12_CHN,Shanghai (China),8817.33 +R12_WEU,Hamburg (Germany),R12_CHN,Shanghai (China),15351.54 +R12_FSU,Arctic LNG 2 (TBC) (Russia),R12_EEU,Swinoujscie (Poland),5022.3 +R12_LAM,DTSE/GEGUA Oil Terminal (Brazil),R12_EEU,Swinoujscie (Poland),11121.92 +R12_MEA,Jeddah (Saudi Arabia),R12_EEU,Swinoujscie (Poland),8852.73 +R12_NAM,Houston (United States),R12_EEU,Swinoujscie (Poland),10043.47 +R12_PAO,Hakodate (Japan),R12_EEU,Swinoujscie (Poland),13383.91 +R12_PAS,Jakarta (Indonesia),R12_EEU,Swinoujscie (Poland),17366.83 +R12_SAS,Jawahar Dweep (India),R12_EEU,Swinoujscie (Poland),13074.08 +R12_WEU,Hamburg (Germany),R12_EEU,Swinoujscie (Poland),1193.9 +R12_LAM,DTSE/GEGUA Oil Terminal (Brazil),R12_FSU,Arctic LNG 2 (TBC) (Russia),14482.85 +R12_MEA,Jeddah (Saudi Arabia),R12_FSU,Arctic LNG 2 (TBC) (Russia),12247.89 +R12_NAM,Houston (United States),R12_FSU,Arctic LNG 2 (TBC) (Russia),12417.93 +R12_PAO,Hakodate (Japan),R12_FSU,Arctic LNG 2 (TBC) (Russia),9106.8 +R12_PAS,Jakarta (Indonesia),R12_FSU,Arctic LNG 2 (TBC) (Russia),15752.79 +R12_SAS,Jawahar Dweep (India),R12_FSU,Arctic LNG 2 (TBC) (Russia),16469.24 +R12_WEU,Hamburg (Germany),R12_FSU,Arctic LNG 2 (TBC) (Russia),4912.22 +R12_MEA,Jeddah (Saudi Arabia),R12_LAM,DTSE/GEGUA Oil Terminal (Brazil),12965.1 +R12_NAM,Houston (United States),R12_LAM,DTSE/GEGUA Oil Terminal (Brazil),10138.75 +R12_PAO,Hakodate (Japan),R12_LAM,DTSE/GEGUA Oil Terminal (Brazil),22158.64 +R12_PAS,Jakarta (Indonesia),R12_LAM,DTSE/GEGUA Oil Terminal (Brazil),15761.96 +R12_SAS,Jawahar Dweep (India),R12_LAM,DTSE/GEGUA Oil Terminal (Brazil),14773.15 +R12_WEU,Hamburg (Germany),R12_LAM,DTSE/GEGUA Oil Terminal (Brazil),10330.03 +R12_NAM,Houston (United States),R12_MEA,Jeddah (Saudi Arabia),13833.28 +R12_PAO,Hakodate (Japan),R12_MEA,Jeddah (Saudi Arabia),14246.32 +R12_PAS,Jakarta (Indonesia),R12_MEA,Jeddah (Saudi Arabia),8683.39 +R12_SAS,Jawahar Dweep (India),R12_MEA,Jeddah (Saudi Arabia),4390.63 +R12_WEU,Hamburg (Germany),R12_MEA,Jeddah (Saudi Arabia),8060.85 +R12_PAO,Hakodate (Japan),R12_NAM,Houston (United States),16793.43 +R12_PAS,Jakarta (Indonesia),R12_NAM,Houston (United States),22347.38 +R12_SAS,Jawahar Dweep (India),R12_NAM,Houston (United States),18054.62 +R12_WEU,Hamburg (Germany),R12_NAM,Houston (United States),9724.35 +R12_PAS,Jakarta (Indonesia),R12_PAO,Hakodate (Japan),6794.95 +R12_SAS,Jawahar Dweep (India),R12_PAO,Hakodate (Japan),10829.08 +R12_WEU,Hamburg (Germany),R12_PAO,Hakodate (Japan),13273.83 +R12_SAS,Jawahar Dweep (India),R12_PAS,Jakarta (Indonesia),5333.67 +R12_WEU,Hamburg (Germany),R12_PAS,Jakarta (Indonesia),16574.95 +R12_WEU,Hamburg (Germany),R12_SAS,Jawahar Dweep (India),12282.19 diff --git a/message_ix_models/data/bilateralize/distances/distances.xlsx b/message_ix_models/data/bilateralize/distances/distances.xlsx new file mode 100644 index 0000000000..fad28143c4 --- /dev/null +++ b/message_ix_models/data/bilateralize/distances/distances.xlsx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2f1a7f1fb0b00f60e43d03d61442b1152c94b0dca032de04b5bf9089e64002d +size 31428 diff --git a/message_ix_models/data/bilateralize/foil_piped/specify_network_foil_piped.csv b/message_ix_models/data/bilateralize/foil_piped/specify_network_foil_piped.csv new file mode 100644 index 0000000000..e5a9c7ad67 --- /dev/null +++ b/message_ix_models/data/bilateralize/foil_piped/specify_network_foil_piped.csv @@ -0,0 +1,133 @@ +exporter,importer,export_technology,import_technology,"INCLUDE? (No=0, Yes=1)" +R12_AFR,R12_RCPA,foil_piped_exp_rcpa,foil_piped_imp,0 +R12_AFR,R12_CHN,foil_piped_exp_chn,foil_piped_imp,0 +R12_AFR,R12_EEU,foil_piped_exp_eeu,foil_piped_imp,0 +R12_AFR,R12_FSU,foil_piped_exp_fsu,foil_piped_imp,0 +R12_AFR,R12_LAM,foil_piped_exp_lam,foil_piped_imp,0 +R12_AFR,R12_MEA,foil_piped_exp_mea,foil_piped_imp,1 +R12_AFR,R12_NAM,foil_piped_exp_nam,foil_piped_imp,0 +R12_AFR,R12_PAO,foil_piped_exp_pao,foil_piped_imp,0 +R12_AFR,R12_PAS,foil_piped_exp_pas,foil_piped_imp,0 +R12_AFR,R12_SAS,foil_piped_exp_sas,foil_piped_imp,0 +R12_AFR,R12_WEU,foil_piped_exp_weu,foil_piped_imp,1 +R12_RCPA,R12_AFR,foil_piped_exp_afr,foil_piped_imp,0 +R12_RCPA,R12_CHN,foil_piped_exp_chn,foil_piped_imp,0 +R12_RCPA,R12_EEU,foil_piped_exp_eeu,foil_piped_imp,0 +R12_RCPA,R12_FSU,foil_piped_exp_fsu,foil_piped_imp,0 +R12_RCPA,R12_LAM,foil_piped_exp_lam,foil_piped_imp,0 +R12_RCPA,R12_MEA,foil_piped_exp_mea,foil_piped_imp,0 +R12_RCPA,R12_NAM,foil_piped_exp_nam,foil_piped_imp,0 +R12_RCPA,R12_PAO,foil_piped_exp_pao,foil_piped_imp,0 +R12_RCPA,R12_PAS,foil_piped_exp_pas,foil_piped_imp,0 +R12_RCPA,R12_SAS,foil_piped_exp_sas,foil_piped_imp,0 +R12_RCPA,R12_WEU,foil_piped_exp_weu,foil_piped_imp,0 +R12_CHN,R12_AFR,foil_piped_exp_afr,foil_piped_imp,0 +R12_CHN,R12_RCPA,foil_piped_exp_rcpa,foil_piped_imp,0 +R12_CHN,R12_EEU,foil_piped_exp_eeu,foil_piped_imp,0 +R12_CHN,R12_FSU,foil_piped_exp_fsu,foil_piped_imp,1 +R12_CHN,R12_LAM,foil_piped_exp_lam,foil_piped_imp,0 +R12_CHN,R12_MEA,foil_piped_exp_mea,foil_piped_imp,0 +R12_CHN,R12_NAM,foil_piped_exp_nam,foil_piped_imp,0 +R12_CHN,R12_PAO,foil_piped_exp_pao,foil_piped_imp,0 +R12_CHN,R12_PAS,foil_piped_exp_pas,foil_piped_imp,0 +R12_CHN,R12_SAS,foil_piped_exp_sas,foil_piped_imp,0 +R12_CHN,R12_WEU,foil_piped_exp_weu,foil_piped_imp,0 +R12_EEU,R12_AFR,foil_piped_exp_afr,foil_piped_imp,0 +R12_EEU,R12_RCPA,foil_piped_exp_rcpa,foil_piped_imp,0 +R12_EEU,R12_CHN,foil_piped_exp_chn,foil_piped_imp,0 +R12_EEU,R12_FSU,foil_piped_exp_fsu,foil_piped_imp,1 +R12_EEU,R12_LAM,foil_piped_exp_lam,foil_piped_imp,0 +R12_EEU,R12_MEA,foil_piped_exp_mea,foil_piped_imp,0 +R12_EEU,R12_NAM,foil_piped_exp_nam,foil_piped_imp,0 +R12_EEU,R12_PAO,foil_piped_exp_pao,foil_piped_imp,0 +R12_EEU,R12_PAS,foil_piped_exp_pas,foil_piped_imp,0 +R12_EEU,R12_SAS,foil_piped_exp_sas,foil_piped_imp,0 +R12_EEU,R12_WEU,foil_piped_exp_weu,foil_piped_imp,1 +R12_FSU,R12_AFR,foil_piped_exp_afr,foil_piped_imp,0 +R12_FSU,R12_RCPA,foil_piped_exp_rcpa,foil_piped_imp,1 +R12_FSU,R12_CHN,foil_piped_exp_chn,foil_piped_imp,1 +R12_FSU,R12_EEU,foil_piped_exp_eeu,foil_piped_imp,1 +R12_FSU,R12_LAM,foil_piped_exp_lam,foil_piped_imp,0 +R12_FSU,R12_MEA,foil_piped_exp_mea,foil_piped_imp,1 +R12_FSU,R12_NAM,foil_piped_exp_nam,foil_piped_imp,0 +R12_FSU,R12_PAO,foil_piped_exp_pao,foil_piped_imp,0 +R12_FSU,R12_PAS,foil_piped_exp_pas,foil_piped_imp,1 +R12_FSU,R12_SAS,foil_piped_exp_sas,foil_piped_imp,1 +R12_FSU,R12_WEU,foil_piped_exp_weu,foil_piped_imp,1 +R12_LAM,R12_AFR,foil_piped_exp_afr,foil_piped_imp,0 +R12_LAM,R12_RCPA,foil_piped_exp_rcpa,foil_piped_imp,0 +R12_LAM,R12_CHN,foil_piped_exp_chn,foil_piped_imp,0 +R12_LAM,R12_EEU,foil_piped_exp_eeu,foil_piped_imp,0 +R12_LAM,R12_FSU,foil_piped_exp_fsu,foil_piped_imp,0 +R12_LAM,R12_MEA,foil_piped_exp_mea,foil_piped_imp,0 +R12_LAM,R12_NAM,foil_piped_exp_nam,foil_piped_imp,1 +R12_LAM,R12_PAO,foil_piped_exp_pao,foil_piped_imp,0 +R12_LAM,R12_PAS,foil_piped_exp_pas,foil_piped_imp,0 +R12_LAM,R12_SAS,foil_piped_exp_sas,foil_piped_imp,0 +R12_LAM,R12_WEU,foil_piped_exp_weu,foil_piped_imp,0 +R12_MEA,R12_AFR,foil_piped_exp_afr,foil_piped_imp,1 +R12_MEA,R12_RCPA,foil_piped_exp_rcpa,foil_piped_imp,0 +R12_MEA,R12_CHN,foil_piped_exp_chn,foil_piped_imp,0 +R12_MEA,R12_EEU,foil_piped_exp_eeu,foil_piped_imp,1 +R12_MEA,R12_FSU,foil_piped_exp_fsu,foil_piped_imp,1 +R12_MEA,R12_LAM,foil_piped_exp_lam,foil_piped_imp,0 +R12_MEA,R12_NAM,foil_piped_exp_nam,foil_piped_imp,0 +R12_MEA,R12_PAO,foil_piped_exp_pao,foil_piped_imp,0 +R12_MEA,R12_PAS,foil_piped_exp_pas,foil_piped_imp,0 +R12_MEA,R12_SAS,foil_piped_exp_sas,foil_piped_imp,1 +R12_MEA,R12_WEU,foil_piped_exp_weu,foil_piped_imp,1 +R12_NAM,R12_AFR,foil_piped_exp_afr,foil_piped_imp,0 +R12_NAM,R12_RCPA,foil_piped_exp_rcpa,foil_piped_imp,0 +R12_NAM,R12_CHN,foil_piped_exp_chn,foil_piped_imp,0 +R12_NAM,R12_EEU,foil_piped_exp_eeu,foil_piped_imp,0 +R12_NAM,R12_FSU,foil_piped_exp_fsu,foil_piped_imp,0 +R12_NAM,R12_LAM,foil_piped_exp_lam,foil_piped_imp,1 +R12_NAM,R12_MEA,foil_piped_exp_mea,foil_piped_imp,0 +R12_NAM,R12_PAO,foil_piped_exp_pao,foil_piped_imp,0 +R12_NAM,R12_PAS,foil_piped_exp_pas,foil_piped_imp,0 +R12_NAM,R12_SAS,foil_piped_exp_sas,foil_piped_imp,0 +R12_NAM,R12_WEU,foil_piped_exp_weu,foil_piped_imp,0 +R12_PAO,R12_AFR,foil_piped_exp_afr,foil_piped_imp,0 +R12_PAO,R12_RCPA,foil_piped_exp_rcpa,foil_piped_imp,0 +R12_PAO,R12_CHN,foil_piped_exp_chn,foil_piped_imp,0 +R12_PAO,R12_EEU,foil_piped_exp_eeu,foil_piped_imp,0 +R12_PAO,R12_FSU,foil_piped_exp_fsu,foil_piped_imp,0 +R12_PAO,R12_LAM,foil_piped_exp_lam,foil_piped_imp,0 +R12_PAO,R12_MEA,foil_piped_exp_mea,foil_piped_imp,0 +R12_PAO,R12_NAM,foil_piped_exp_nam,foil_piped_imp,0 +R12_PAO,R12_PAS,foil_piped_exp_pas,foil_piped_imp,0 +R12_PAO,R12_SAS,foil_piped_exp_sas,foil_piped_imp,0 +R12_PAO,R12_WEU,foil_piped_exp_weu,foil_piped_imp,0 +R12_PAS,R12_AFR,foil_piped_exp_afr,foil_piped_imp,0 +R12_PAS,R12_RCPA,foil_piped_exp_rcpa,foil_piped_imp,1 +R12_PAS,R12_CHN,foil_piped_exp_chn,foil_piped_imp,1 +R12_PAS,R12_EEU,foil_piped_exp_eeu,foil_piped_imp,0 +R12_PAS,R12_FSU,foil_piped_exp_fsu,foil_piped_imp,1 +R12_PAS,R12_LAM,foil_piped_exp_lam,foil_piped_imp,0 +R12_PAS,R12_MEA,foil_piped_exp_mea,foil_piped_imp,0 +R12_PAS,R12_NAM,foil_piped_exp_nam,foil_piped_imp,0 +R12_PAS,R12_PAO,foil_piped_exp_pao,foil_piped_imp,0 +R12_PAS,R12_SAS,foil_piped_exp_sas,foil_piped_imp,0 +R12_PAS,R12_WEU,foil_piped_exp_weu,foil_piped_imp,0 +R12_SAS,R12_AFR,foil_piped_exp_afr,foil_piped_imp,0 +R12_SAS,R12_RCPA,foil_piped_exp_rcpa,foil_piped_imp,0 +R12_SAS,R12_CHN,foil_piped_exp_chn,foil_piped_imp,0 +R12_SAS,R12_EEU,foil_piped_exp_eeu,foil_piped_imp,0 +R12_SAS,R12_FSU,foil_piped_exp_fsu,foil_piped_imp,1 +R12_SAS,R12_LAM,foil_piped_exp_lam,foil_piped_imp,0 +R12_SAS,R12_MEA,foil_piped_exp_mea,foil_piped_imp,1 +R12_SAS,R12_NAM,foil_piped_exp_nam,foil_piped_imp,0 +R12_SAS,R12_PAO,foil_piped_exp_pao,foil_piped_imp,0 +R12_SAS,R12_PAS,foil_piped_exp_pas,foil_piped_imp,1 +R12_SAS,R12_WEU,foil_piped_exp_weu,foil_piped_imp,0 +R12_WEU,R12_AFR,foil_piped_exp_afr,foil_piped_imp,1 +R12_WEU,R12_RCPA,foil_piped_exp_rcpa,foil_piped_imp,0 +R12_WEU,R12_CHN,foil_piped_exp_chn,foil_piped_imp,0 +R12_WEU,R12_EEU,foil_piped_exp_eeu,foil_piped_imp,1 +R12_WEU,R12_FSU,foil_piped_exp_fsu,foil_piped_imp,1 +R12_WEU,R12_LAM,foil_piped_exp_lam,foil_piped_imp,0 +R12_WEU,R12_MEA,foil_piped_exp_mea,foil_piped_imp,1 +R12_WEU,R12_NAM,foil_piped_exp_nam,foil_piped_imp,0 +R12_WEU,R12_PAO,foil_piped_exp_pao,foil_piped_imp,0 +R12_WEU,R12_PAS,foil_piped_exp_pas,foil_piped_imp,0 +R12_WEU,R12_SAS,foil_piped_exp_sas,foil_piped_imp,0 diff --git a/message_ix_models/data/bilateralize/gas_piped/specify_network_gas_piped.csv b/message_ix_models/data/bilateralize/gas_piped/specify_network_gas_piped.csv new file mode 100644 index 0000000000..a419969801 --- /dev/null +++ b/message_ix_models/data/bilateralize/gas_piped/specify_network_gas_piped.csv @@ -0,0 +1,133 @@ +exporter,importer,export_technology,import_technology,"INCLUDE? (No=0, Yes=1)" +R12_AFR,R12_RCPA,gas_piped_exp_rcpa,gas_piped_imp,0 +R12_AFR,R12_CHN,gas_piped_exp_chn,gas_piped_imp,0 +R12_AFR,R12_EEU,gas_piped_exp_eeu,gas_piped_imp,0 +R12_AFR,R12_FSU,gas_piped_exp_fsu,gas_piped_imp,0 +R12_AFR,R12_LAM,gas_piped_exp_lam,gas_piped_imp,0 +R12_AFR,R12_MEA,gas_piped_exp_mea,gas_piped_imp,1 +R12_AFR,R12_NAM,gas_piped_exp_nam,gas_piped_imp,0 +R12_AFR,R12_PAO,gas_piped_exp_pao,gas_piped_imp,0 +R12_AFR,R12_PAS,gas_piped_exp_pas,gas_piped_imp,0 +R12_AFR,R12_SAS,gas_piped_exp_sas,gas_piped_imp,0 +R12_AFR,R12_WEU,gas_piped_exp_weu,gas_piped_imp,1 +R12_RCPA,R12_AFR,gas_piped_exp_afr,gas_piped_imp,0 +R12_RCPA,R12_CHN,gas_piped_exp_chn,gas_piped_imp,0 +R12_RCPA,R12_EEU,gas_piped_exp_eeu,gas_piped_imp,0 +R12_RCPA,R12_FSU,gas_piped_exp_fsu,gas_piped_imp,0 +R12_RCPA,R12_LAM,gas_piped_exp_lam,gas_piped_imp,0 +R12_RCPA,R12_MEA,gas_piped_exp_mea,gas_piped_imp,0 +R12_RCPA,R12_NAM,gas_piped_exp_nam,gas_piped_imp,0 +R12_RCPA,R12_PAO,gas_piped_exp_pao,gas_piped_imp,0 +R12_RCPA,R12_PAS,gas_piped_exp_pas,gas_piped_imp,0 +R12_RCPA,R12_SAS,gas_piped_exp_sas,gas_piped_imp,0 +R12_RCPA,R12_WEU,gas_piped_exp_weu,gas_piped_imp,0 +R12_CHN,R12_AFR,gas_piped_exp_afr,gas_piped_imp,0 +R12_CHN,R12_RCPA,gas_piped_exp_rcpa,gas_piped_imp,0 +R12_CHN,R12_EEU,gas_piped_exp_eeu,gas_piped_imp,0 +R12_CHN,R12_FSU,gas_piped_exp_fsu,gas_piped_imp,1 +R12_CHN,R12_LAM,gas_piped_exp_lam,gas_piped_imp,0 +R12_CHN,R12_MEA,gas_piped_exp_mea,gas_piped_imp,0 +R12_CHN,R12_NAM,gas_piped_exp_nam,gas_piped_imp,0 +R12_CHN,R12_PAO,gas_piped_exp_pao,gas_piped_imp,0 +R12_CHN,R12_PAS,gas_piped_exp_pas,gas_piped_imp,0 +R12_CHN,R12_SAS,gas_piped_exp_sas,gas_piped_imp,0 +R12_CHN,R12_WEU,gas_piped_exp_weu,gas_piped_imp,0 +R12_EEU,R12_AFR,gas_piped_exp_afr,gas_piped_imp,0 +R12_EEU,R12_RCPA,gas_piped_exp_rcpa,gas_piped_imp,0 +R12_EEU,R12_CHN,gas_piped_exp_chn,gas_piped_imp,0 +R12_EEU,R12_FSU,gas_piped_exp_fsu,gas_piped_imp,1 +R12_EEU,R12_LAM,gas_piped_exp_lam,gas_piped_imp,0 +R12_EEU,R12_MEA,gas_piped_exp_mea,gas_piped_imp,0 +R12_EEU,R12_NAM,gas_piped_exp_nam,gas_piped_imp,0 +R12_EEU,R12_PAO,gas_piped_exp_pao,gas_piped_imp,0 +R12_EEU,R12_PAS,gas_piped_exp_pas,gas_piped_imp,0 +R12_EEU,R12_SAS,gas_piped_exp_sas,gas_piped_imp,0 +R12_EEU,R12_WEU,gas_piped_exp_weu,gas_piped_imp,1 +R12_FSU,R12_AFR,gas_piped_exp_afr,gas_piped_imp,0 +R12_FSU,R12_RCPA,gas_piped_exp_rcpa,gas_piped_imp,1 +R12_FSU,R12_CHN,gas_piped_exp_chn,gas_piped_imp,1 +R12_FSU,R12_EEU,gas_piped_exp_eeu,gas_piped_imp,1 +R12_FSU,R12_LAM,gas_piped_exp_lam,gas_piped_imp,0 +R12_FSU,R12_MEA,gas_piped_exp_mea,gas_piped_imp,1 +R12_FSU,R12_NAM,gas_piped_exp_nam,gas_piped_imp,0 +R12_FSU,R12_PAO,gas_piped_exp_pao,gas_piped_imp,0 +R12_FSU,R12_PAS,gas_piped_exp_pas,gas_piped_imp,1 +R12_FSU,R12_SAS,gas_piped_exp_sas,gas_piped_imp,1 +R12_FSU,R12_WEU,gas_piped_exp_weu,gas_piped_imp,1 +R12_LAM,R12_AFR,gas_piped_exp_afr,gas_piped_imp,0 +R12_LAM,R12_RCPA,gas_piped_exp_rcpa,gas_piped_imp,0 +R12_LAM,R12_CHN,gas_piped_exp_chn,gas_piped_imp,0 +R12_LAM,R12_EEU,gas_piped_exp_eeu,gas_piped_imp,0 +R12_LAM,R12_FSU,gas_piped_exp_fsu,gas_piped_imp,0 +R12_LAM,R12_MEA,gas_piped_exp_mea,gas_piped_imp,0 +R12_LAM,R12_NAM,gas_piped_exp_nam,gas_piped_imp,1 +R12_LAM,R12_PAO,gas_piped_exp_pao,gas_piped_imp,0 +R12_LAM,R12_PAS,gas_piped_exp_pas,gas_piped_imp,0 +R12_LAM,R12_SAS,gas_piped_exp_sas,gas_piped_imp,0 +R12_LAM,R12_WEU,gas_piped_exp_weu,gas_piped_imp,0 +R12_MEA,R12_AFR,gas_piped_exp_afr,gas_piped_imp,1 +R12_MEA,R12_RCPA,gas_piped_exp_rcpa,gas_piped_imp,0 +R12_MEA,R12_CHN,gas_piped_exp_chn,gas_piped_imp,0 +R12_MEA,R12_EEU,gas_piped_exp_eeu,gas_piped_imp,1 +R12_MEA,R12_FSU,gas_piped_exp_fsu,gas_piped_imp,1 +R12_MEA,R12_LAM,gas_piped_exp_lam,gas_piped_imp,0 +R12_MEA,R12_NAM,gas_piped_exp_nam,gas_piped_imp,0 +R12_MEA,R12_PAO,gas_piped_exp_pao,gas_piped_imp,0 +R12_MEA,R12_PAS,gas_piped_exp_pas,gas_piped_imp,0 +R12_MEA,R12_SAS,gas_piped_exp_sas,gas_piped_imp,1 +R12_MEA,R12_WEU,gas_piped_exp_weu,gas_piped_imp,1 +R12_NAM,R12_AFR,gas_piped_exp_afr,gas_piped_imp,0 +R12_NAM,R12_RCPA,gas_piped_exp_rcpa,gas_piped_imp,0 +R12_NAM,R12_CHN,gas_piped_exp_chn,gas_piped_imp,0 +R12_NAM,R12_EEU,gas_piped_exp_eeu,gas_piped_imp,0 +R12_NAM,R12_FSU,gas_piped_exp_fsu,gas_piped_imp,0 +R12_NAM,R12_LAM,gas_piped_exp_lam,gas_piped_imp,1 +R12_NAM,R12_MEA,gas_piped_exp_mea,gas_piped_imp,0 +R12_NAM,R12_PAO,gas_piped_exp_pao,gas_piped_imp,0 +R12_NAM,R12_PAS,gas_piped_exp_pas,gas_piped_imp,0 +R12_NAM,R12_SAS,gas_piped_exp_sas,gas_piped_imp,0 +R12_NAM,R12_WEU,gas_piped_exp_weu,gas_piped_imp,0 +R12_PAO,R12_AFR,gas_piped_exp_afr,gas_piped_imp,0 +R12_PAO,R12_RCPA,gas_piped_exp_rcpa,gas_piped_imp,0 +R12_PAO,R12_CHN,gas_piped_exp_chn,gas_piped_imp,0 +R12_PAO,R12_EEU,gas_piped_exp_eeu,gas_piped_imp,0 +R12_PAO,R12_FSU,gas_piped_exp_fsu,gas_piped_imp,0 +R12_PAO,R12_LAM,gas_piped_exp_lam,gas_piped_imp,0 +R12_PAO,R12_MEA,gas_piped_exp_mea,gas_piped_imp,0 +R12_PAO,R12_NAM,gas_piped_exp_nam,gas_piped_imp,0 +R12_PAO,R12_PAS,gas_piped_exp_pas,gas_piped_imp,0 +R12_PAO,R12_SAS,gas_piped_exp_sas,gas_piped_imp,0 +R12_PAO,R12_WEU,gas_piped_exp_weu,gas_piped_imp,0 +R12_PAS,R12_AFR,gas_piped_exp_afr,gas_piped_imp,0 +R12_PAS,R12_RCPA,gas_piped_exp_rcpa,gas_piped_imp,1 +R12_PAS,R12_CHN,gas_piped_exp_chn,gas_piped_imp,1 +R12_PAS,R12_EEU,gas_piped_exp_eeu,gas_piped_imp,0 +R12_PAS,R12_FSU,gas_piped_exp_fsu,gas_piped_imp,1 +R12_PAS,R12_LAM,gas_piped_exp_lam,gas_piped_imp,0 +R12_PAS,R12_MEA,gas_piped_exp_mea,gas_piped_imp,0 +R12_PAS,R12_NAM,gas_piped_exp_nam,gas_piped_imp,0 +R12_PAS,R12_PAO,gas_piped_exp_pao,gas_piped_imp,0 +R12_PAS,R12_SAS,gas_piped_exp_sas,gas_piped_imp,0 +R12_PAS,R12_WEU,gas_piped_exp_weu,gas_piped_imp,0 +R12_SAS,R12_AFR,gas_piped_exp_afr,gas_piped_imp,0 +R12_SAS,R12_RCPA,gas_piped_exp_rcpa,gas_piped_imp,0 +R12_SAS,R12_CHN,gas_piped_exp_chn,gas_piped_imp,0 +R12_SAS,R12_EEU,gas_piped_exp_eeu,gas_piped_imp,0 +R12_SAS,R12_FSU,gas_piped_exp_fsu,gas_piped_imp,1 +R12_SAS,R12_LAM,gas_piped_exp_lam,gas_piped_imp,0 +R12_SAS,R12_MEA,gas_piped_exp_mea,gas_piped_imp,1 +R12_SAS,R12_NAM,gas_piped_exp_nam,gas_piped_imp,0 +R12_SAS,R12_PAO,gas_piped_exp_pao,gas_piped_imp,0 +R12_SAS,R12_PAS,gas_piped_exp_pas,gas_piped_imp,1 +R12_SAS,R12_WEU,gas_piped_exp_weu,gas_piped_imp,0 +R12_WEU,R12_AFR,gas_piped_exp_afr,gas_piped_imp,1 +R12_WEU,R12_RCPA,gas_piped_exp_rcpa,gas_piped_imp,0 +R12_WEU,R12_CHN,gas_piped_exp_chn,gas_piped_imp,0 +R12_WEU,R12_EEU,gas_piped_exp_eeu,gas_piped_imp,1 +R12_WEU,R12_FSU,gas_piped_exp_fsu,gas_piped_imp,1 +R12_WEU,R12_LAM,gas_piped_exp_lam,gas_piped_imp,0 +R12_WEU,R12_MEA,gas_piped_exp_mea,gas_piped_imp,1 +R12_WEU,R12_NAM,gas_piped_exp_nam,gas_piped_imp,0 +R12_WEU,R12_PAO,gas_piped_exp_pao,gas_piped_imp,0 +R12_WEU,R12_PAS,gas_piped_exp_pas,gas_piped_imp,0 +R12_WEU,R12_SAS,gas_piped_exp_sas,gas_piped_imp,0 diff --git a/message_ix_models/data/bilateralize/loil_piped/specify_network_loil_piped.csv b/message_ix_models/data/bilateralize/loil_piped/specify_network_loil_piped.csv new file mode 100644 index 0000000000..38a00d963d --- /dev/null +++ b/message_ix_models/data/bilateralize/loil_piped/specify_network_loil_piped.csv @@ -0,0 +1,133 @@ +exporter,importer,export_technology,import_technology,"INCLUDE? (No=0, Yes=1)" +R12_AFR,R12_RCPA,loil_piped_exp_rcpa,loil_piped_imp,0 +R12_AFR,R12_CHN,loil_piped_exp_chn,loil_piped_imp,0 +R12_AFR,R12_EEU,loil_piped_exp_eeu,loil_piped_imp,0 +R12_AFR,R12_FSU,loil_piped_exp_fsu,loil_piped_imp,0 +R12_AFR,R12_LAM,loil_piped_exp_lam,loil_piped_imp,0 +R12_AFR,R12_MEA,loil_piped_exp_mea,loil_piped_imp,1 +R12_AFR,R12_NAM,loil_piped_exp_nam,loil_piped_imp,0 +R12_AFR,R12_PAO,loil_piped_exp_pao,loil_piped_imp,0 +R12_AFR,R12_PAS,loil_piped_exp_pas,loil_piped_imp,0 +R12_AFR,R12_SAS,loil_piped_exp_sas,loil_piped_imp,0 +R12_AFR,R12_WEU,loil_piped_exp_weu,loil_piped_imp,1 +R12_RCPA,R12_AFR,loil_piped_exp_afr,loil_piped_imp,0 +R12_RCPA,R12_CHN,loil_piped_exp_chn,loil_piped_imp,0 +R12_RCPA,R12_EEU,loil_piped_exp_eeu,loil_piped_imp,0 +R12_RCPA,R12_FSU,loil_piped_exp_fsu,loil_piped_imp,0 +R12_RCPA,R12_LAM,loil_piped_exp_lam,loil_piped_imp,0 +R12_RCPA,R12_MEA,loil_piped_exp_mea,loil_piped_imp,0 +R12_RCPA,R12_NAM,loil_piped_exp_nam,loil_piped_imp,0 +R12_RCPA,R12_PAO,loil_piped_exp_pao,loil_piped_imp,0 +R12_RCPA,R12_PAS,loil_piped_exp_pas,loil_piped_imp,0 +R12_RCPA,R12_SAS,loil_piped_exp_sas,loil_piped_imp,0 +R12_RCPA,R12_WEU,loil_piped_exp_weu,loil_piped_imp,0 +R12_CHN,R12_AFR,loil_piped_exp_afr,loil_piped_imp,0 +R12_CHN,R12_RCPA,loil_piped_exp_rcpa,loil_piped_imp,0 +R12_CHN,R12_EEU,loil_piped_exp_eeu,loil_piped_imp,0 +R12_CHN,R12_FSU,loil_piped_exp_fsu,loil_piped_imp,1 +R12_CHN,R12_LAM,loil_piped_exp_lam,loil_piped_imp,0 +R12_CHN,R12_MEA,loil_piped_exp_mea,loil_piped_imp,0 +R12_CHN,R12_NAM,loil_piped_exp_nam,loil_piped_imp,0 +R12_CHN,R12_PAO,loil_piped_exp_pao,loil_piped_imp,0 +R12_CHN,R12_PAS,loil_piped_exp_pas,loil_piped_imp,0 +R12_CHN,R12_SAS,loil_piped_exp_sas,loil_piped_imp,0 +R12_CHN,R12_WEU,loil_piped_exp_weu,loil_piped_imp,0 +R12_EEU,R12_AFR,loil_piped_exp_afr,loil_piped_imp,0 +R12_EEU,R12_RCPA,loil_piped_exp_rcpa,loil_piped_imp,0 +R12_EEU,R12_CHN,loil_piped_exp_chn,loil_piped_imp,0 +R12_EEU,R12_FSU,loil_piped_exp_fsu,loil_piped_imp,1 +R12_EEU,R12_LAM,loil_piped_exp_lam,loil_piped_imp,0 +R12_EEU,R12_MEA,loil_piped_exp_mea,loil_piped_imp,0 +R12_EEU,R12_NAM,loil_piped_exp_nam,loil_piped_imp,0 +R12_EEU,R12_PAO,loil_piped_exp_pao,loil_piped_imp,0 +R12_EEU,R12_PAS,loil_piped_exp_pas,loil_piped_imp,0 +R12_EEU,R12_SAS,loil_piped_exp_sas,loil_piped_imp,0 +R12_EEU,R12_WEU,loil_piped_exp_weu,loil_piped_imp,1 +R12_FSU,R12_AFR,loil_piped_exp_afr,loil_piped_imp,0 +R12_FSU,R12_RCPA,loil_piped_exp_rcpa,loil_piped_imp,1 +R12_FSU,R12_CHN,loil_piped_exp_chn,loil_piped_imp,1 +R12_FSU,R12_EEU,loil_piped_exp_eeu,loil_piped_imp,1 +R12_FSU,R12_LAM,loil_piped_exp_lam,loil_piped_imp,0 +R12_FSU,R12_MEA,loil_piped_exp_mea,loil_piped_imp,1 +R12_FSU,R12_NAM,loil_piped_exp_nam,loil_piped_imp,0 +R12_FSU,R12_PAO,loil_piped_exp_pao,loil_piped_imp,0 +R12_FSU,R12_PAS,loil_piped_exp_pas,loil_piped_imp,1 +R12_FSU,R12_SAS,loil_piped_exp_sas,loil_piped_imp,1 +R12_FSU,R12_WEU,loil_piped_exp_weu,loil_piped_imp,1 +R12_LAM,R12_AFR,loil_piped_exp_afr,loil_piped_imp,0 +R12_LAM,R12_RCPA,loil_piped_exp_rcpa,loil_piped_imp,0 +R12_LAM,R12_CHN,loil_piped_exp_chn,loil_piped_imp,0 +R12_LAM,R12_EEU,loil_piped_exp_eeu,loil_piped_imp,0 +R12_LAM,R12_FSU,loil_piped_exp_fsu,loil_piped_imp,0 +R12_LAM,R12_MEA,loil_piped_exp_mea,loil_piped_imp,0 +R12_LAM,R12_NAM,loil_piped_exp_nam,loil_piped_imp,1 +R12_LAM,R12_PAO,loil_piped_exp_pao,loil_piped_imp,0 +R12_LAM,R12_PAS,loil_piped_exp_pas,loil_piped_imp,0 +R12_LAM,R12_SAS,loil_piped_exp_sas,loil_piped_imp,0 +R12_LAM,R12_WEU,loil_piped_exp_weu,loil_piped_imp,0 +R12_MEA,R12_AFR,loil_piped_exp_afr,loil_piped_imp,1 +R12_MEA,R12_RCPA,loil_piped_exp_rcpa,loil_piped_imp,0 +R12_MEA,R12_CHN,loil_piped_exp_chn,loil_piped_imp,0 +R12_MEA,R12_EEU,loil_piped_exp_eeu,loil_piped_imp,1 +R12_MEA,R12_FSU,loil_piped_exp_fsu,loil_piped_imp,1 +R12_MEA,R12_LAM,loil_piped_exp_lam,loil_piped_imp,0 +R12_MEA,R12_NAM,loil_piped_exp_nam,loil_piped_imp,0 +R12_MEA,R12_PAO,loil_piped_exp_pao,loil_piped_imp,0 +R12_MEA,R12_PAS,loil_piped_exp_pas,loil_piped_imp,0 +R12_MEA,R12_SAS,loil_piped_exp_sas,loil_piped_imp,1 +R12_MEA,R12_WEU,loil_piped_exp_weu,loil_piped_imp,1 +R12_NAM,R12_AFR,loil_piped_exp_afr,loil_piped_imp,0 +R12_NAM,R12_RCPA,loil_piped_exp_rcpa,loil_piped_imp,0 +R12_NAM,R12_CHN,loil_piped_exp_chn,loil_piped_imp,0 +R12_NAM,R12_EEU,loil_piped_exp_eeu,loil_piped_imp,0 +R12_NAM,R12_FSU,loil_piped_exp_fsu,loil_piped_imp,0 +R12_NAM,R12_LAM,loil_piped_exp_lam,loil_piped_imp,1 +R12_NAM,R12_MEA,loil_piped_exp_mea,loil_piped_imp,0 +R12_NAM,R12_PAO,loil_piped_exp_pao,loil_piped_imp,0 +R12_NAM,R12_PAS,loil_piped_exp_pas,loil_piped_imp,0 +R12_NAM,R12_SAS,loil_piped_exp_sas,loil_piped_imp,0 +R12_NAM,R12_WEU,loil_piped_exp_weu,loil_piped_imp,0 +R12_PAO,R12_AFR,loil_piped_exp_afr,loil_piped_imp,0 +R12_PAO,R12_RCPA,loil_piped_exp_rcpa,loil_piped_imp,0 +R12_PAO,R12_CHN,loil_piped_exp_chn,loil_piped_imp,0 +R12_PAO,R12_EEU,loil_piped_exp_eeu,loil_piped_imp,0 +R12_PAO,R12_FSU,loil_piped_exp_fsu,loil_piped_imp,0 +R12_PAO,R12_LAM,loil_piped_exp_lam,loil_piped_imp,0 +R12_PAO,R12_MEA,loil_piped_exp_mea,loil_piped_imp,0 +R12_PAO,R12_NAM,loil_piped_exp_nam,loil_piped_imp,0 +R12_PAO,R12_PAS,loil_piped_exp_pas,loil_piped_imp,0 +R12_PAO,R12_SAS,loil_piped_exp_sas,loil_piped_imp,0 +R12_PAO,R12_WEU,loil_piped_exp_weu,loil_piped_imp,0 +R12_PAS,R12_AFR,loil_piped_exp_afr,loil_piped_imp,0 +R12_PAS,R12_RCPA,loil_piped_exp_rcpa,loil_piped_imp,1 +R12_PAS,R12_CHN,loil_piped_exp_chn,loil_piped_imp,1 +R12_PAS,R12_EEU,loil_piped_exp_eeu,loil_piped_imp,0 +R12_PAS,R12_FSU,loil_piped_exp_fsu,loil_piped_imp,1 +R12_PAS,R12_LAM,loil_piped_exp_lam,loil_piped_imp,0 +R12_PAS,R12_MEA,loil_piped_exp_mea,loil_piped_imp,0 +R12_PAS,R12_NAM,loil_piped_exp_nam,loil_piped_imp,0 +R12_PAS,R12_PAO,loil_piped_exp_pao,loil_piped_imp,0 +R12_PAS,R12_SAS,loil_piped_exp_sas,loil_piped_imp,0 +R12_PAS,R12_WEU,loil_piped_exp_weu,loil_piped_imp,0 +R12_SAS,R12_AFR,loil_piped_exp_afr,loil_piped_imp,0 +R12_SAS,R12_RCPA,loil_piped_exp_rcpa,loil_piped_imp,0 +R12_SAS,R12_CHN,loil_piped_exp_chn,loil_piped_imp,0 +R12_SAS,R12_EEU,loil_piped_exp_eeu,loil_piped_imp,0 +R12_SAS,R12_FSU,loil_piped_exp_fsu,loil_piped_imp,1 +R12_SAS,R12_LAM,loil_piped_exp_lam,loil_piped_imp,0 +R12_SAS,R12_MEA,loil_piped_exp_mea,loil_piped_imp,1 +R12_SAS,R12_NAM,loil_piped_exp_nam,loil_piped_imp,0 +R12_SAS,R12_PAO,loil_piped_exp_pao,loil_piped_imp,0 +R12_SAS,R12_PAS,loil_piped_exp_pas,loil_piped_imp,1 +R12_SAS,R12_WEU,loil_piped_exp_weu,loil_piped_imp,0 +R12_WEU,R12_AFR,loil_piped_exp_afr,loil_piped_imp,1 +R12_WEU,R12_RCPA,loil_piped_exp_rcpa,loil_piped_imp,0 +R12_WEU,R12_CHN,loil_piped_exp_chn,loil_piped_imp,0 +R12_WEU,R12_EEU,loil_piped_exp_eeu,loil_piped_imp,1 +R12_WEU,R12_FSU,loil_piped_exp_fsu,loil_piped_imp,1 +R12_WEU,R12_LAM,loil_piped_exp_lam,loil_piped_imp,0 +R12_WEU,R12_MEA,loil_piped_exp_mea,loil_piped_imp,1 +R12_WEU,R12_NAM,loil_piped_exp_nam,loil_piped_imp,0 +R12_WEU,R12_PAO,loil_piped_exp_pao,loil_piped_imp,0 +R12_WEU,R12_PAS,loil_piped_exp_pas,loil_piped_imp,0 +R12_WEU,R12_SAS,loil_piped_exp_sas,loil_piped_imp,0 diff --git a/message_ix_models/data/bilateralize/specific_energy.xlsx b/message_ix_models/data/bilateralize/specific_energy.xlsx new file mode 100644 index 0000000000..c3ff6ddd56 --- /dev/null +++ b/message_ix_models/data/bilateralize/specific_energy.xlsx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:102688a98a0ef2d2fec5d3eb0508eb0153120e9b650f6caa30047c1d0b938476 +size 11936 diff --git a/message_ix_models/tests/tools/bilateralize/__init__.py b/message_ix_models/tests/tools/bilateralize/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/message_ix_models/tests/tools/bilateralize/test_bare_to_scenario.py b/message_ix_models/tests/tools/bilateralize/test_bare_to_scenario.py new file mode 100644 index 0000000000..4557113e6b --- /dev/null +++ b/message_ix_models/tests/tools/bilateralize/test_bare_to_scenario.py @@ -0,0 +1,39 @@ +import pytest + +from message_ix_models.tools.bilateralize.bare_to_scenario import ( + build_parameter_sheets, + calibrate_historical_shipping, +) +from message_ix_models.tools.bilateralize.utils import get_logger, load_config + +MARK = pytest.mark.xfail( + raises=FileNotFoundError, + reason="Input data files not available for testing.", +) + + +@MARK # P:/ene.model/MESSAGE_trade/IMO/GISIS/Crude Tankers.csv +def test_calibrate_historical_shipping( + project_name: str | None = None, config_name: str | None = None +) -> None: + config, config_path, tec_config = load_config( + project_name=project_name, config_name=config_name, load_tec_config=True + ) + + covered_tec = config["covered_trade_technologies"] + + # Get logger + log = get_logger(__name__) + + # Read and inflate sheets based on model horizon + trade_dict = build_parameter_sheets( + log=log, project_name=project_name, config_name=config_name + ) + calibrate_historical_shipping( + config=config, + trade_dict=trade_dict, + covered_tec=covered_tec, + project_name=project_name, + config_name=config_name, + ) + assert True diff --git a/message_ix_models/tests/tools/bilateralize/test_historical_calibration.py b/message_ix_models/tests/tools/bilateralize/test_historical_calibration.py new file mode 100644 index 0000000000..0efa457dd8 --- /dev/null +++ b/message_ix_models/tests/tools/bilateralize/test_historical_calibration.py @@ -0,0 +1,105 @@ +import pandas as pd +import pytest + +from message_ix_models.tools.bilateralize.historical_calibration import ( + build_hist_new_capacity_flow, + build_hist_new_capacity_trade, + build_historical_activity, + build_historical_price, + convert_trade, + generate_cfdict, + import_iea_balances, + import_iea_gas, + import_uncomtrade, + reformat_to_parameter, + setup_datapath, +) + +MARK = pytest.mark.xfail( + raises=FileNotFoundError, + reason="Input data files not available for testing.", +) + + +@pytest.fixture +def message_regions() -> str: + """`message_regions` parameter to some functions.""" + return "R12" + + +@MARK # IMO/GISIS/* Tankers.csv +@pytest.mark.parametrize( + "infile, ship_type", + # Values appearing where the function is used in bare_to_scenario() + [ + ("Crude Tankers.csv", "crudeoil_tanker_loil"), + ("LH2 Tankers.csv", "lh2_tanker_loil"), + ("LNG Tankers.csv", "LNG_Tanker_loil"), + ("LNG Tankers.csv", "LNG_Tanker_LNG"), + ("Oil Tankers.csv", "oil_tanker_eth"), + ("Oil Tankers.csv", "oil_tanker_foil"), + ("Oil Tankers.csv", "oil_tanker_loil"), + ], +) +def test_build_hist_new_capacity_flow(infile: str, ship_type: str) -> None: + build_hist_new_capacity_flow(infile, ship_type) + + +@MARK # UN Comtrade/BACI/shortenedBACI.csv +def test_build_hist_new_capacity_trade() -> None: + build_hist_new_capacity_trade() + + +@MARK # UN Comtrade/BACI/shortenedBACI.csv +def test_build_historical_activity() -> None: + build_historical_activity() + + +@MARK # UN Comtrade/BACI/shortenedBACI.csv +def test_build_historical_price() -> None: + build_historical_price() + + +# @MARK # IEA/WEB2025/WEB_TRADEFLOWS.txt +# def test_check_iea_balances() -> None: +# indf = pd.DataFrame() +# check_iea_balances(indf) + + +@MARK # UN Comtrade/BACI/shortenedBACI.csv +def test_convert_trade(message_regions: str) -> None: + convert_trade(message_regions) + + +@MARK # IEA/WEB2025/CONV.txt +def test_generate_cfdict(message_regions: str) -> None: + generate_cfdict(message_regions) + + +@MARK # IEA/WEB2025/EARLYBIG1.txt +def test_import_iea_balances() -> None: + import_iea_balances() + + +@MARK # IEA/NATGAS/WIMPDAT.txt +def test_import_iea_gas() -> None: + import_iea_gas() + + +@MARK # UN Comtrade/BACI/BACI_HS92_Y2005_V202501.csv +def test_import_uncomtrade() -> None: + import_uncomtrade() + + +@pytest.mark.xfail(raises=UnboundLocalError, reason="Test input data is empty.") +def test_reformat_to_parameter(message_regions: str) -> None: + # Column names that must be present on `indf` + # TODO Mention these in the function docstring + indf = pd.DataFrame(columns=["IMPORTER", "EXPORTER", "MESSAGE COMMODITY", "YEAR"]) + parameter_name = "foo" + reformat_to_parameter(indf, message_regions, parameter_name) + + +def test_setup_datapath() -> None: + setup_datapath() + # TODO Extend with assertions diff --git a/message_ix_models/tests/tools/bilateralize/test_mariteam_calibration.py b/message_ix_models/tests/tools/bilateralize/test_mariteam_calibration.py new file mode 100644 index 0000000000..68ec266347 --- /dev/null +++ b/message_ix_models/tests/tools/bilateralize/test_mariteam_calibration.py @@ -0,0 +1,32 @@ +import pytest + +from message_ix_models.tools.bilateralize.mariteam_calibration import ( + calibrate_mariteam, +) +from message_ix_models.tools.bilateralize.utils import load_config + +MARK = pytest.mark.xfail( + raises=FileNotFoundError, + reason="Input data files not available for testing.", +) + + +@MARK # P:/ene.model/MESSAGE_trade/MariTEAM/MariTEAM_output_2025-07-21.csv +def test_calibrate_mariteam( + project_name: str | None = None, + config_name: str | None = None, +) -> None: + config, config_path, tec_config = load_config( + project_name=project_name, config_name=config_name, load_tec_config=True + ) + + covered_tec = config["covered_trade_technologies"] + + calibrate_mariteam( + covered_tec=covered_tec, + message_regions="R12", + project_name=project_name, + config_name=config_name, + ) + + assert True diff --git a/message_ix_models/tests/tools/bilateralize/test_pull_gem.py b/message_ix_models/tests/tools/bilateralize/test_pull_gem.py new file mode 100644 index 0000000000..ba91311eb0 --- /dev/null +++ b/message_ix_models/tests/tools/bilateralize/test_pull_gem.py @@ -0,0 +1,29 @@ +import pytest + +from message_ix_models.tools.bilateralize.pull_gem import gem_region, import_gem + + +def test_gem_region() -> None: + gem_region() + + +@pytest.mark.xfail( + raises=FileNotFoundError, + reason="Input files not available for testing: " + "Global Energy Monitor/GEM-GOIT-*.csv", +) +@pytest.mark.parametrize( + "input_file", + # Values appearing where the function is called in prepare_edit_files() + [ + ("GEM-GGIT-Gas-Pipelines-2024-12.csv"), + ("GEM-GOIT-Oil-NGL-Pipelines-2025-03.csv"), + ], +) +def test_import_gem(input_file: str) -> None: + import_gem( + input_file=input_file, + trade_technology="foo", + flow_technology="bar", + flow_commodity="baz", + ) diff --git a/message_ix_models/tests/tools/test_bilateralize.py b/message_ix_models/tests/tools/test_bilateralize.py new file mode 100644 index 0000000000..47960a604d --- /dev/null +++ b/message_ix_models/tests/tools/test_bilateralize.py @@ -0,0 +1,126 @@ +""" +Test the bilateralize tool +""" + +# Import packages +import os +from pathlib import Path + +import pytest + +from message_ix_models import Context, testing +from message_ix_models.tools.bilateralize.bare_to_scenario import bare_to_scenario +from message_ix_models.tools.bilateralize.load_and_solve import ( + add_trade_parameters, + add_trade_sets, + remove_pao_coal_constraint, + remove_trade_tech, + update_additional_parameters, + update_bunker_fuels, + update_relation_parameters, +) +from message_ix_models.tools.bilateralize.prepare_edit import ( + prepare_edit_files, +) +from message_ix_models.tools.bilateralize.utils import get_logger, load_config +from message_ix_models.util import package_data_path + +# Get logger +log = get_logger(__name__) + + +def test_generate_edit_files(access_to_p: bool = False): + """ + Checks that required files are generated in both + edit_files and bare_files directories. + """ + config_base, config_path, config_tec = load_config( + project_name=None, config_name=None, load_tec_config=True + ) + data_path = package_data_path("bilateralize") + data_path = Path(os.path.join(os.path.dirname(data_path), "bilateralize")) + + prepare_edit_files(project_name=None, config_name=None, P_access=access_to_p) + + req_files = ["input", "output", "technical_lifetime", "capacity_factor"] + + for tec in config_tec.keys(): + for file in req_files: + assert os.path.isfile( + os.path.join(data_path, tec, "edit_files", file + ".csv") + ) + assert os.path.isfile( + os.path.join(data_path, tec, "bare_files", file + ".csv") + ) + + +def test_build_parameter_sheets(): + """ + Checks that dictionary of parameter dataframes is built correctly. + """ + config_base, config_path, config_tec = load_config( + project_name=None, config_name=None, load_tec_config=True + ) + + test_dict = bare_to_scenario( + project_name=None, config_name=None, p_drive_access=False + ) + + assert isinstance(test_dict, dict) + assert len(test_dict) == len(config_tec) + for tec in config_tec.keys(): + assert isinstance(test_dict[tec], dict) + assert "trade" in test_dict[tec].keys() + assert "flow" in test_dict[tec].keys() + + +def test_bilat_scenario(request: pytest.FixtureRequest, test_context: Context): + """ + Test bilateralization on a scenario. + """ + config_base, config_path, config_tec = load_config( + project_name=None, config_name=None, load_tec_config=True + ) + + # Set up test scenario + test_context.model.regions = "R12" + scen = testing.bare_res(request, test_context) + + # Add unit + mp = scen.platform + mp.add_unit("Mt-km") + mp.add_unit("USD/Mt-km") + + # Add global node + with scen.transact("Add global node"): + scen.add_set("node", "R12_GLB") + + # Set up bilateralization dictionary + bilat_dict = bare_to_scenario( + project_name=None, config_name=None, p_drive_access=False + ) + + covered_tec = config_base.get("covered_trade_technologies") + for tec in covered_tec: + # Remove existing technologies related to trade + remove_trade_tech(scen=scen, log=log, config_tec=config_tec, tec=tec) + + # Add to sets: technology, level, commodity, mode + add_trade_sets(scen=scen, log=log, trade_dict=bilat_dict, tec=tec) + + # Add parameters + add_trade_parameters(scen=scen, log=log, trade_dict=bilat_dict, tec=tec) + + # Relation activity, upper, and lower + update_relation_parameters(scen=scen, log=log, trade_dict=bilat_dict, tec=tec) + + # Update bunker fuels + update_bunker_fuels(scen=scen, tec=tec, log=log, config_tec=config_tec) + + # Update additional parameters + update_additional_parameters(scen=scen, extra_parameter_updates=None) + + # Remove PAO coal and gas constraints on MESSAGEix-GLOBIOM (won't run in test) + remove_pao_coal_constraint(scen=scen, log=log, MESSAGEix_GLOBIOM=False) + + assert True diff --git a/message_ix_models/tools/bilateralize/__init__.py b/message_ix_models/tools/bilateralize/__init__.py new file mode 100644 index 0000000000..cb543ff9c9 --- /dev/null +++ b/message_ix_models/tools/bilateralize/__init__.py @@ -0,0 +1,4 @@ +"""Bilateral trade. + +See :doc:`/api/tools-bilateralize`. +""" diff --git a/message_ix_models/tools/bilateralize/bare_to_scenario.py b/message_ix_models/tools/bilateralize/bare_to_scenario.py new file mode 100644 index 0000000000..3f2e0a0c09 --- /dev/null +++ b/message_ix_models/tools/bilateralize/bare_to_scenario.py @@ -0,0 +1,433 @@ +# -*- coding: utf-8 -*- +""" +Move data from bare files to a dictionary to update a MESSAGEix scenario + +This script is the second step in implementing the bilateralize tool. +It moves data from /data/bilateralize/[your_trade_commodity]/bare_files/ +to a dictionary compatible with updating a MESSAGEix scenario. +""" + +# Import packages +import logging +import os +import pickle +from pathlib import Path +from typing import Any, Dict + +import numpy as np +import pandas as pd + +from message_ix_models.tools.bilateralize.historical_calibration import ( + build_hist_new_capacity_flow, + build_hist_new_capacity_trade, + build_historical_activity, +) +from message_ix_models.tools.bilateralize.utils import get_logger, load_config +from message_ix_models.util import package_data_path + + +# %% Broadcast vintage years +def broadcast_years( + df: pd.DataFrame, year_type: str, year_list: list[int] +) -> pd.DataFrame: + """ + Broadcast vintage, relation, or activity years. + + Args: + df: Input parameter DataFrame + year_type: Type of year to broadcast (e.g., 'year_vtg', 'year_rel', 'year_act') + year_list: List of years to broadcast + Returns: + pd.DataFrame: DataFrame with expanded rows for each year + """ + all_new_rows = [] + for _, row in df.iterrows(): + for y in year_list: + new_row = row.copy() + new_row[year_type] = int(y) + all_new_rows.append(new_row) + result_df = pd.concat([df, pd.DataFrame(all_new_rows)], ignore_index=True) + result_df = result_df[result_df[year_type] != "broadcast"] + return result_df.drop_duplicates() + + +# %% Broadcast years to create vintage-activity year pairs. +def broadcast_yv_ya( + df: pd.DataFrame, ya_list: list[int], yv_list: list[int], tec_lifetime: pd.DataFrame +): + """ + Broadcast years to create vintage-activity year pairs. + + Args: + df: Input parameter DataFrame + ya_list: List of activity years to consider + yv_list: List of vintage years to consider + tec_lifetime: Technical lifetime of the technology, provided via dataframe + Returns: + pd.DataFrame: DataFrame with expanded rows for each vintage-activity year pair + """ + all_new_rows = [] + + tecltdf = tec_lifetime.copy() + tecltdf["teclt"] = tecltdf["value"] + + lts = df.merge( + tecltdf[["node_loc", "technology", "teclt"]].drop_duplicates(), + left_on=["node_loc", "technology"], + right_on=["node_loc", "technology"], + how="left", + ) + + # Process each row in the original DataFrame + for _, row in lts.iterrows(): + teclt_row = row["teclt"] + + # For each activity year + for ya in ya_list: + # Get all vintage years that are <=year_act for a period= ya - teclt_row] + + # Create new rows for each vintage year + for yv in yv_list: + new_row = row.copy() + new_row["year_act"] = int(ya) + new_row["year_vtg"] = int(yv) + all_new_rows.append(new_row) + + # Combine original DataFrame with new rows + result_df = pd.DataFrame(all_new_rows).drop(["teclt"], axis=1) + result_df = result_df[result_df["year_vtg"] != "broadcast"] + return result_df + + +# %% Full broadcast function +def full_broadcast( + data_dict: dict, + tec: str, + ty: str, + ya_list: list[int], + yv_list: list[int], + log: logging.Logger, +) -> dict: + """ + Full broadcast function + Args: + data_dict: Dictionary of parameter dataframes + tec: Technology name + ty: Type of parameter (trade or flow) + ya_list: List of activity years + yv_list: List of vintage years + log: Logger + Outputs: + data_dict: Dictionary of parameter dataframes with broadcasted years + """ + for i in data_dict[ty].keys(): + if "year_rel" in data_dict[ty][i].columns: + log.info(f"Parameter {i} in {tec} {ty} broadcasted for year_rel") + if data_dict[ty][i]["year_rel"].iloc[0] == "broadcast": + data_dict[ty][i] = broadcast_years( + df=data_dict[ty][i], year_type="year_rel", year_list=ya_list + ) + data_dict[ty][i]["year_act"] = data_dict[ty][i]["year_rel"] + else: + pass + + if ( + "year_vtg" in data_dict[ty][i].columns + and "year_act" in data_dict[ty][i].columns + ): + if ( + data_dict[ty][i]["year_vtg"].iloc[0] == "broadcast" + and data_dict[ty][i]["year_act"].iloc[0] == "broadcast" + ): + log.info(f"{i} in {tec} {ty} broadcasted for year_vtg+year_act") + teclt_df = data_dict[ty]["technical_lifetime"].copy() + data_dict[ty][i] = broadcast_yv_ya( + df=data_dict[ty][i], + ya_list=ya_list, + yv_list=yv_list, + tec_lifetime=teclt_df, + ) + elif ( + data_dict[ty][i]["year_vtg"].iloc[0] == "broadcast" + and data_dict[ty][i]["year_act"].iloc[0] != "broadcast" + ): + log.info(f"{i} in {tec} {ty} broadcasted for year_vtg") + data_dict[ty][i] = broadcast_years( + df=data_dict[ty][i], year_type="year_vtg", year_list=yv_list + ) + elif ( + "year_vtg" in data_dict[ty][i].columns + and "year_act" not in data_dict[ty][i].columns + ): + if data_dict[ty][i]["year_vtg"].iloc[0] == "broadcast": + log.info(f"{i} in {tec} {ty} broadcasted for year_vtg") + data_dict[ty][i] = broadcast_years( + df=data_dict[ty][i], year_type="year_vtg", year_list=yv_list + ) + elif ( + "year_vtg" not in data_dict[ty][i].columns + and "year_act" in data_dict[ty][i].columns + ): + if data_dict[ty][i]["year_act"].iloc[0] == "broadcast": + log.info(f"{i} in {tec} {ty} broadcasted for year_act") + data_dict[ty][i] = broadcast_years( + df=data_dict[ty][i], year_type="year_act", year_list=ya_list + ) + else: + pass + return data_dict + + +# %% Build out bare sheets +def build_parameter_sheets( + log, project_name: str | None = None, config_name: str | None = None +): + """ + Read the input csv files and build the tech sets and parameters. + + Args: + project_name (str, optional): Project name (message_ix_models/project/[THIS]) + config_name (str, optional): Name of the config file. + If None, uses default config from data/bilateralize/config_default.yaml + Returns: + outdict: Dictionary of parameter dataframes + """ + # Load config + config, config_path = load_config(project_name, config_name) + + covered_tec = config.get("covered_trade_technologies", {}) + + outdict = dict() + + ya_list = config["timeframes"]["year_act_list"] + yv_list = config["timeframes"]["year_vtg_list"] + + for tec in covered_tec: + tecpath = os.path.join(Path(package_data_path("bilateralize")), tec) + + data_dict: Dict[str, Dict[str, Any]] = {"trade": {}, "flow": {}} + + for ty in ["trade", "flow"]: + if ty == "trade": + tpath = os.path.join(tecpath, "bare_files") + elif ty == "flow": + tpath = os.path.join(tecpath, "bare_files", "flow_technology") + + csv_files = [f for f in Path(tpath).glob("*.csv")] + + for csv_file in csv_files: + key = csv_file.stem + data_dict[ty][key] = pd.read_csv(csv_file) + + # Broadcast the data + for ty in ["trade", "flow"]: + data_dict = full_broadcast( + data_dict=data_dict, + tec=tec, + ty=ty, + ya_list=ya_list, + yv_list=yv_list, + log=log, + ) + + # Imports do not vintage + for par in ["capacity_factor", "input", "output"]: + if par in list(data_dict["trade"].keys()): + vdf = data_dict["trade"][par] + vdf = vdf[ + ( + (vdf["technology"].str.contains("_imp")) + & (vdf["year_vtg"] == vdf["year_act"]) + ) + | (vdf["technology"].str.contains("_exp_")) + ] + data_dict["trade"][par] = vdf + + # Variable costs for flows should not broadcast + for par in ["var_cost"]: + if par in list(data_dict["flow"].keys()): + vdf = data_dict["flow"][par] + vdf = vdf[vdf["year_act"] == vdf["year_vtg"]] + data_dict["flow"][par] = vdf + + outdict[tec] = data_dict + + return outdict + + +def calibrate_historical_shipping( + config: dict, + trade_dict: dict, + covered_tec: list[str], + project_name: str | None = None, + config_name: str | None = None, +): + # Historical new capacity for maritime shipping + shipping_fuel_dict = config["shipping_fuels"] + # TODO: Add coal + hist_cr_loil = build_hist_new_capacity_flow( + infile="Crude Tankers.csv", + ship_type="crudeoil_tanker_loil", + project_name=project_name, + config_name=config_name, + ) + hist_lh2_loil = build_hist_new_capacity_flow( + infile="LH2 Tankers.csv", + ship_type="lh2_tanker_loil", + project_name=project_name, + config_name=config_name, + ) + hist_lng = pd.DataFrame() + for f in ["loil", "LNG"]: + hist_lng_f = build_hist_new_capacity_flow( + infile="LNG Tankers.csv", + ship_type="LNG_tanker_" + f, + project_name=project_name, + config_name=config_name, + ) + hist_lng_f["value"] *= shipping_fuel_dict["LNG_tanker"]["LNG_tanker_" + f] + hist_lng = pd.concat([hist_lng, hist_lng_f]) + + hist_oil = pd.DataFrame() + for f in ["loil", "foil", "eth"]: + hist_oil_f = build_hist_new_capacity_flow( + infile="Oil Tankers.csv", + ship_type="oil_tanker_" + f, + project_name=project_name, + config_name=config_name, + ) + hist_oil_f["value"] *= shipping_fuel_dict["oil_tanker"]["oil_tanker_" + f] + hist_oil = pd.concat([hist_oil, hist_oil_f]) + hist_eth = hist_oil[hist_oil["technology"] != "oil_tanker_foil"] + + nc_dict = { + "crudeoil_shipped": hist_cr_loil, + "lh2_shipped": hist_lh2_loil, + "LNG_shipped": hist_lng, + "eth_shipped": hist_eth, + "foil_shipped": hist_oil, + "loil_shipped": hist_oil, + } + for tec in nc_dict.keys(): + trade_dict[tec]["flow"]["historical_new_capacity"] = nc_dict[tec] + + # Historical activity should only be added for technologies in input + for tec in covered_tec: + input_tecs = trade_dict[tec]["trade"]["input"]["technology"] + + if "historical_activity" in trade_dict[tec]["trade"].keys(): + tdf = trade_dict[tec]["trade"]["historical_activity"] + tdf = tdf[tdf["technology"].isin(input_tecs)] + trade_dict[tec]["trade"]["historical_activity"] = tdf + + if "historical_new_capacity" in trade_dict[tec]["trade"].keys(): + tdf = trade_dict[tec]["trade"]["historical_new_capacity"] + tdf = tdf[tdf["technology"].isin(input_tecs)] + trade_dict[tec]["trade"]["historical_new_capacity"] = tdf + + return trade_dict + + +def bare_to_scenario( + project_name: str | None = None, + config_name: str | None = None, + scenario_parameter_name: str = "scenario_parameters.pkl", + p_drive_access: bool = False, +): + """ + Move data from bare files to a dictionary to update a MESSAGEix scenario + + Args: + project_name: Name of the project (e.g., 'newpathways') + config_name: Name of the config file (e.g., 'config.yaml') + scenario_parameter_name: Name of the scenario parameter file + + Output: + trade_dict: Dictionary compatible with updating a MESSAGEix scenario + """ + # Bring in configuration + config, config_path, tec_config = load_config( + project_name=project_name, config_name=config_name, load_tec_config=True + ) + + covered_tec = config["covered_trade_technologies"] + message_regions = config["scenario"]["regions"] + + # Get logger + log = get_logger(__name__) + + # Read and inflate sheets based on model horizon + trade_dict = build_parameter_sheets( + log=log, project_name=project_name, config_name=config_name + ) + + if p_drive_access: + # Historical calibration for trade technology + histdf = build_historical_activity( + message_regions=message_regions, + project_name=project_name, + config_name=config_name, + reimport_BACI=False, + ) + histdf.to_csv("check.csv") + histdf = histdf[histdf["year_act"].isin([2000, 2005, 2010, 2015, 2020, 2023])] + histdf["year_act"] = np.where( + (histdf["year_act"] == 2023), + 2025, # TODO: 2023 to 2025 only for now + histdf["year_act"], + ) + histdf = histdf[histdf["value"] > 0] + histdf["technology"] = histdf["technology"].str.replace("ethanol_", "eth_") + histdf["technology"] = histdf["technology"].str.replace("fueloil_", "foil_") + + histnc = build_hist_new_capacity_trade( + message_regions=message_regions, + project_name=project_name, + config_name=config_name, + ) + + hist_tec = {} + for tec in [ + c + for c in covered_tec + if c not in ["crudeoil_piped", "foil_piped", "loil_piped"] + ]: + add_tec = tec_config[tec][tec + "_trade"]["trade_technology"] + "_exp" + hist_tec[tec] = add_tec + + for tec in hist_tec.keys(): + log.info("Add historical activity for " + tec) + add_df = histdf[histdf["technology"].str.contains(hist_tec[tec])] + trade_dict[tec]["trade"]["historical_activity"] = add_df + + log.info("Add historical new capacity for " + tec) + add_df = histnc[histnc["technology"].str.contains(hist_tec[tec])] + trade_dict[tec]["trade"]["historical_new_capacity"] = add_df + + trade_dict = calibrate_historical_shipping( + config=config, + trade_dict=trade_dict, + covered_tec=covered_tec, + project_name=project_name, + config_name=config_name, + ) + + # Ensure flow technologies are only added once + covered_flow_tec: list[str] = [] + for tec in covered_tec: + if "input" in list(trade_dict[tec]["flow"].keys()): + flow_tecs = list(trade_dict[tec]["flow"]["input"]["technology"].unique()) + for par in trade_dict[tec]["flow"].keys(): + trade_dict[tec]["flow"][par] = trade_dict[tec]["flow"][par][ + ~trade_dict[tec]["flow"][par]["technology"].isin(covered_flow_tec) + ] + covered_flow_tec = covered_flow_tec + flow_tecs + + # Save trade_dictionary + tdf = os.path.join(os.path.dirname(config_path), scenario_parameter_name) + with open(tdf, "wb") as file_handler: + pickle.dump(trade_dict, file_handler) + + return trade_dict diff --git a/message_ix_models/tools/bilateralize/calculate_distance.py b/message_ix_models/tools/bilateralize/calculate_distance.py new file mode 100644 index 0000000000..d1c067b096 --- /dev/null +++ b/message_ix_models/tools/bilateralize/calculate_distance.py @@ -0,0 +1,145 @@ +""" +Calculate distances between pairs of ports +""" + +import math +import os +from itertools import combinations + +import pandas as pd +from scgraph.geographs.marnet import marnet_geograph + +from message_ix_models.util import package_data_path + + +def haversine_distance(lat1: float, lon1: float, lat2: float, lon2: float) -> float: + """ + Calculate the great circle distance between two points + on the earth (specified in decimal degrees) using the Haversine formula. + + Args: + lat1: Latitude of the first point + lon1: Longitude of the first point + lat2: Latitude of the second point + lon2: Longitude of the second point + Outputs: + Distance in kilometers + """ + # Convert decimal degrees to radians + lat1, lon1, lat2, lon2 = map(math.radians, [lat1, lon1, lat2, lon2]) + + # Haversine formula + dlat = lat2 - lat1 + dlon = lon2 - lon1 + a = ( + math.sin(dlat / 2) ** 2 + + math.cos(lat1) * math.cos(lat2) * math.sin(dlon / 2) ** 2 + ) + c = 2 * math.asin(math.sqrt(a)) + + # Radius of earth in kilometers + r = 6371 + + return c * r + + +def calculate_port_distances(df: pd.DataFrame) -> pd.DataFrame: + """ + Read CSV file with port data and calculate distances between all port combinations. + + Args: + df: DataFrame containing Port, Latitude, Longitude columns + + Outputs: + DataFrame with columns 'Port1', 'Port2', 'Distance_km' + """ + # Check if required columns exist + required_columns = ["Port", "Latitude", "Longitude"] + missing_columns = [col for col in required_columns if col not in df.columns] + + if missing_columns: + raise ValueError(f"Missing required columns: {missing_columns}") + + # Remove rows with missing coordinates + ports_clean = df.dropna(subset=["Latitude", "Longitude"]) + + if ports_clean.empty: + raise ValueError("No valid coordinate data found in the file") + + # Get all combinations of ports (without repetition) + port_combinations = list(combinations(ports_clean.index, 2)) + print(f"Calculating distances for {len(port_combinations)} port pairs...") + + # Calculate distances between all port combinations + distances = [] + for i, j in port_combinations: + port1 = ports_clean.iloc[i] + port2 = ports_clean.iloc[j] + + distance = marnet_geograph.get_shortest_path( + origin_node={ + "latitude": port1["Latitude"], + "longitude": port1["Longitude"], + }, + destination_node={ + "latitude": port2["Latitude"], + "longitude": port2["Longitude"], + }, + ) + + distances.append( + { + "Port1": port1["Port"], + "Port2": port2["Port"], + "Distance_km": round(distance["length"], 2), + } + ) + + # Create DataFrame with results + outdf1 = pd.DataFrame(distances) + + # Concatenate other direction too + outdf2 = outdf1.copy() + outdf2 = outdf2.rename(columns={"Port1": "Port2", "Port2": "Port1"}) + outdf = pd.concat([outdf1, outdf2]) + return outdf + + +def calculate_distance(regional_specification: str = "R12"): + """ + Run distance calculation. + + Args: + regional_specification: MESSAGE regional specification (e.g., "R12") + Outputs: + CSV file in data/bilateralize/distances/ that includes + distances for regional specification + """ + # Specify the path to CSV file + csv_path = os.path.abspath( + os.path.join( + os.path.dirname(package_data_path("bilateralize")), + "bilateralize", + "distances", + ) + ) + + infile = pd.read_excel( + os.path.join(csv_path, "distances.xlsx"), sheet_name="node_ports" + ) + infile = infile[infile["Regionalization"] == regional_specification] + + # Calculate distances + df = calculate_port_distances(infile) + + # Add regions back + for i in ["1", "2"]: + df = df.merge( + infile[["Node", "Port"]], left_on="Port" + i, right_on="Port", how="left" + ) + df = df.rename(columns={"Node": "Node" + i}) + df = df[["Node1", "Port1", "Node2", "Port2", "Distance_km"]] + + df.to_csv( + os.path.join(csv_path, regional_specification + "_distances.csv"), index=False + ) diff --git a/message_ix_models/tools/bilateralize/historical_calibration.py b/message_ix_models/tools/bilateralize/historical_calibration.py new file mode 100644 index 0000000000..942ff30287 --- /dev/null +++ b/message_ix_models/tools/bilateralize/historical_calibration.py @@ -0,0 +1,935 @@ +# -*- coding: utf-8 -*- +""" +Historical Calibration +""" + +# Import packages +import os +import pickle + +import message_ix +import numpy as np +import pandas as pd +import yaml + +from message_ix_models.tools.bilateralize.utils import load_config +from message_ix_models.util import package_data_path + +# Reimport large files? +reimport_IEA = False +reimport_BACI = False + + +# Set up data paths +def setup_datapath(project_name: str | None = None, config_name: str | None = None): + """ + Set up data paths. + + Args: + project_name: Name of project + config_name: Name of config file + Outputs: + data_paths: Dictionary of data paths + """ + # Pull in configuration + config, config_path = load_config( + project_name=project_name, config_name=config_name + ) + p_drive = config["p_drive_location"] + + # Data paths + data_path = os.path.join(p_drive, "MESSAGE_trade") + iea_path = os.path.join(data_path, "IEA") + iea_diagnostics_path = os.path.join(iea_path, "diagnostics") + iea_web_path = os.path.join(iea_path, "WEB2025") + iea_gas_path = os.path.join(iea_path, "NATGAS") + baci_path = os.path.join(data_path, "UN Comtrade", "BACI") + imo_path = os.path.join(data_path, "IMO") + cw_path = os.path.join(data_path, "crosswalks") + + data_paths = dict( + iea_web=iea_web_path, + iea_gas=iea_gas_path, + iea_diag=iea_diagnostics_path, + baci=baci_path, + imo=imo_path, + cw=cw_path, + ) + + return data_paths + + +# Dictionaries of ISO - IEA - MESSAGE Regions +def generate_cfdict( + message_regions: str, + project_name: str | None = None, + config_name: str | None = None, +): + """ + Generate conversion factor dictionary. + + Args: + message_regions: Regional resolution + project_name: Name of project (e.g., 'newpathways') + config_name: Name of config file + """ + dict_dir = package_data_path("node", message_regions + ".yaml") + with open(dict_dir, "r", encoding="utf8") as f: + dict_message_regions = yaml.safe_load(f) + region_list = [i for i in list(dict_message_regions.keys()) if i != "World"] + + data_paths = setup_datapath(project_name=project_name, config_name=config_name) + + print("Import conversion factors") + cfdf = pd.read_csv( + os.path.join(data_paths["iea_web"], "CONV.txt"), + sep=r"\s+", + header=None, + encoding="windows-1252", + ) + cfdf.columns = ["units", "country", "commodity", "metric", "year", "value"] + cfdf = cfdf[cfdf["year"] > 1990] + cfdf = cfdf[cfdf["units"] == "KJKG"] # KJ/KG + cfdf = cfdf[cfdf["metric"].isin(["NAVERAGE", "NINDPROD"])] # Mean NCV and NCV + cfdf = cfdf[cfdf["value"] != "x"] + + cfdf["conversion (TJ/t)"] = (cfdf["value"].astype(float) * 1000) * (1e-9) + + cf_out = ( + cfdf.groupby(["country", "commodity"])[["conversion (TJ/t)"]] + .mean() + .reset_index() + ) + + # Link ISO codes + cf_cw = pd.read_csv(os.path.join(data_paths["iea_web"], "CONV_country_codes.csv")) + cf_out = cf_out.merge(cf_cw, left_on="country", right_on="IEA COUNTRY", how="inner") + + regvar = message_regions + "_REGION" + cf_out[regvar] = "" + for k in region_list: + if "child" in dict_message_regions[k].keys(): + cf_out[regvar] = np.where( + cf_out["ISO"].isin(dict_message_regions[k]["child"]), k, cf_out[regvar] + ) + + print("Collapse conversion factors to REGION level") + cf_region = ( + cf_out.groupby([regvar, "commodity"])[["conversion (TJ/t)"]] + .mean() + .reset_index() + ) + + print("Collapse conversion factors to FUEL level") + cf_fuel = cf_out.groupby(["commodity"])["conversion (TJ/t)"].mean().reset_index() + + print("Clean up ISO level data") + cf_iso = cf_out[["ISO", "R12_REGION", "commodity", "conversion (TJ/t)"]].copy() + + print("Save conversion factors") + cf_region.to_csv(os.path.join(data_paths["iea_web"], "conv_by_region.csv")) + cf_fuel.to_csv(os.path.join(data_paths["iea_web"], "conv_by_fuel.csv")) + cf_iso.to_csv(os.path.join(data_paths["iea_web"], "conv_by_iso.csv")) + + print("Full dictionaries") + full_dict = {message_regions: cf_region, "fuel": cf_fuel, "ISO": cf_iso} + + picklepath = os.path.join(data_paths["iea_web"], "conversion_factors.pickle") + with open(picklepath, "wb") as file_handler: + pickle.dump(full_dict, file_handler) + + +# Import UN Comtrade data and link to conversion factors +# This does not include natural gas pipelines or LNG, which are from IEA +def import_uncomtrade( + update_year: int = 2024, + project_name: str | None = None, + config_name: str | None = None, +): + """ + Import UN Comtrade data and link to conversion factors, save as CSV and pickle. + + Args: + update_year: Year of last data update + project_name: Name of project (e.g., 'newpathways') + config_name: Name of config file + """ + dict_dir = package_data_path("bilateralize", "commodity_codes.yaml") + with open(dict_dir, "r", encoding="utf8") as f: + commodity_codes = yaml.safe_load(f) + + full_hs_list: list[str] = [] + for c in commodity_codes.keys(): + full_hs_list = full_hs_list + commodity_codes[c]["HS"] + + data_paths = setup_datapath(project_name=project_name, config_name=config_name) + print("Build BACI") + df = pd.DataFrame() + for y in list(range(2005, update_year, 1)): + print("Importing BACI" + str(y)) + ydf = pd.read_csv( + os.path.join(data_paths["baci"], "BACI_HS92_Y" + str(y) + "_V202501.csv"), + encoding="windows-1252", + ) + ydf["k"] = ydf["k"].astype(str).str.zfill(6) + ydf["hs4"] = ydf["k"].str[0:4] + ydf["hs5"] = ydf["k"].str[0:5] + ydf["hs6"] = ydf["k"].str[0:6] + ydf = ydf[ + (ydf["hs4"].isin(full_hs_list)) + | (ydf["hs5"].isin(full_hs_list)) + | (ydf["hs6"].isin(full_hs_list)) + ].copy() + df = pd.concat([df, ydf]) + + print("Save pickle") + picklepath = os.path.join(data_paths["baci"], "full_2005-2023.pickle") + with open(picklepath, "wb") as file_handler: + pickle.dump(df, file_handler) + + df["MESSAGE Commodity"] = "" + for c in commodity_codes.keys(): + df["MESSAGE Commodity"] = np.where( + (df["hs4"].isin(commodity_codes[c]["HS"])) + | (df["hs5"].isin(commodity_codes[c]["HS"])) + | (df["hs6"].isin(commodity_codes[c]["HS"])), + commodity_codes[c]["MESSAGE Commodity"], + df["MESSAGE Commodity"], + ) + + countrycw = pd.read_csv( + os.path.join(data_paths["baci"], "country_codes_V202401b.csv") + ) + df = df.merge( + countrycw[["country_code", "country_iso3"]], + left_on="i", + right_on="country_code", + how="left", + ) + df = df.rename(columns={"country_iso3": "i_iso3"}) + df = df.merge( + countrycw[["country_code", "country_iso3"]], + left_on="j", + right_on="country_code", + how="left", + ) + df = df.rename(columns={"country_iso3": "j_iso3"}) + df = df[["t", "i", "j", "i_iso3", "j_iso3", "k", "MESSAGE Commodity", "v", "q"]] + + df.to_csv(os.path.join(data_paths["baci"], "shortenedBACI.csv")) + + +# Convert trade values +def convert_trade( + message_regions: str, + project_name: str | None = None, + config_name: str | None = None, +): + """ + Convert trade values to energy units. + + Args: + message_regions: Regional resolution + project_name: Name of project (e.g., 'newpathways') + config_name: Name of config file + """ + data_paths = setup_datapath(project_name=project_name, config_name=config_name) + + df = pd.read_csv(os.path.join(data_paths["baci"], "shortenedBACI.csv")) + + with open( + os.path.join(data_paths["iea_web"], "conversion_factors.pickle"), "rb" + ) as file_handler: + conversion_factors = pickle.load(file_handler) + with open( + os.path.join(data_paths["iea_web"], "CONV_addl.yaml"), "r", encoding="utf8" + ) as file_handler: + conversion_addl = yaml.safe_load(file_handler) + cf_codes = pd.read_csv(os.path.join(data_paths["iea_web"], "CONV_hs.csv")) + + df["k"] = df["k"].astype(str) + cf_codes["HS"] = cf_codes["HS"].astype(str) + + df["HS"] = "" + for hs in [i for i in cf_codes["HS"] if len(i) == 4]: # 4 digit HS + df["HS"] = np.where(df["k"].str[0:4] == hs, hs, df["HS"]) + df["HS"] = np.where(df["HS"] == "", df["k"], df["HS"]) + + # Add MESSAGE regions + dict_dir = package_data_path("node", message_regions + ".yaml") + with open(dict_dir, "r", encoding="utf8") as f: + dict_message_regions = yaml.safe_load(f) + region_list = [i for i in list(dict_message_regions.keys()) if i != "World"] + + df["MESSAGE Region"] = "" + for r in region_list: + df["MESSAGE Region"] = np.where( + df["i_iso3"].isin(dict_message_regions[r]["child"]), r, df["MESSAGE Region"] + ) + # Add IEA conversion factors + df = df.merge(cf_codes, left_on="HS", right_on="HS", how="left") + + df = df.merge( + conversion_factors["ISO"][["ISO", "commodity", "conversion (TJ/t)"]], + left_on=["i_iso3", "IEA CONV Commodity"], + right_on=["ISO", "commodity"], + how="left", + ) + df = df.rename(columns={"conversion (TJ/t)": "ISO conversion (TJ/t)"}) + + df = df.merge( + conversion_factors[message_regions], + left_on=["MESSAGE Region", "IEA CONV Commodity"], + right_on=[message_regions + "_REGION", "commodity"], + how="left", + ) + df = df.rename(columns={"conversion (TJ/t)": "Region conversion (TJ/t)"}) + + df = df.merge( + conversion_factors["fuel"], + left_on=["IEA CONV Commodity"], + right_on=["commodity"], + how="left", + ) + df = df.rename(columns={"conversion (TJ/t)": "Fuel conversion (TJ/t)"}) + + df["conversion (TJ/t)"] = df["ISO conversion (TJ/t)"] + df["conversion (TJ/t)"] = np.where( + df["conversion (TJ/t)"].isnull(), + df["Region conversion (TJ/t)"], + df["conversion (TJ/t)"], + ) + df["conversion (TJ/t)"] = df["ISO conversion (TJ/t)"] + df["conversion (TJ/t)"] = np.where( + df["conversion (TJ/t)"].isnull(), + df["Fuel conversion (TJ/t)"], + df["conversion (TJ/t)"], + ) + + df = df[ + [ + "t", + "i", + "j", + "i_iso3", + "j_iso3", + "k", + "MESSAGE Commodity", + "v", + "q", + "conversion (TJ/t)", + ] + ] + + # Add additional conversion factors if missing + for f in conversion_addl.keys(): + df["conversion (TJ/t)"] = np.where( + (df["conversion (TJ/t)"].isnull()) & (df["MESSAGE Commodity"] == f), + conversion_addl[f], + df["conversion (TJ/t)"], + ) + + # Convert to energy units + df["conversion (TJ/t)"] = df["conversion (TJ/t)"].astype(float) + + df = df.rename( + columns={ + "t": "YEAR", + "i_iso3": "EXPORTER", + "j_iso3": "IMPORTER", + "k": "HS", + "v": "VALUE (1000USD)", + "q": "WEIGHT (t)", + "MESSAGE Commodity": "MESSAGE COMMODITY", + } + ) + df["WEIGHT (t)"] = df["WEIGHT (t)"].astype(str) + df = df[~df["WEIGHT (t)"].str.contains("NA")] + df["WEIGHT (t)"] = df["WEIGHT (t)"].astype(float) + df["ENERGY (TJ)"] = df["WEIGHT (t)"] * df["conversion (TJ/t)"] + + df = df[ + [ + "YEAR", + "EXPORTER", + "IMPORTER", + "HS", + "MESSAGE COMMODITY", + "ENERGY (TJ)", + "VALUE (1000USD)", + ] + ] + + return df + + +# Import IEA for LNG and pipeline gas +def import_iea_gas(project_name: str | None = None, config_name: str | None = None): + """ + Import IEA data for LNG and pipeline gas. + + Args: + project_name: Name of project (e.g., 'newpathways') + config_name: Name of config file + """ + data_paths = setup_datapath(project_name=project_name, config_name=config_name) + + ngd = pd.read_csv(os.path.join(data_paths["iea_gas"], "WIMPDAT.txt"), sep=r"\s+") + ngd.columns = ["YEAR", "PRODUCT", "IMPORTER", "EXPORTER", "VALUE"] + + ngd = ngd[ngd["YEAR"] > 1989] # Keep after 1990 only + ngd = ngd[ngd["PRODUCT"].isin(["LNGTJ", "PIPETJ"])] # Keep only TJ values + + ngd["ENERGY (TJ)"] = np.where( + ngd["VALUE"].isin(["..", "x", "c"]), np.nan, ngd["VALUE"] + ) + ngd["ENERGY (TJ)"] = ngd["ENERGY (TJ)"].astype(float) + + ngd["MESSAGE COMMODITY"] = "" + ngd["MESSAGE COMMODITY"] = np.where( + ngd["PRODUCT"] == "LNGTJ", "LNG_shipped", ngd["MESSAGE COMMODITY"] + ) + ngd["MESSAGE COMMODITY"] = np.where( + ngd["PRODUCT"] == "PIPETJ", "gas_piped", ngd["MESSAGE COMMODITY"] + ) + + cf_cw = pd.read_excel(os.path.join(data_paths["cw"], "country_crosswalk.xlsx")) + cf_cw = cf_cw[["message_country", "iea_country"]] + for t in ["EXPORTER", "IMPORTER"]: + ngd = ngd.merge(cf_cw, left_on=t, right_on="iea_country", how="left") + ngd[t] = ngd["message_country"] + ngd = ngd.drop(["iea_country", "message_country"], axis=1) + ngd = ngd[~ngd[t].isnull()] + + ngd = ngd[["YEAR", "EXPORTER", "IMPORTER", "MESSAGE COMMODITY", "ENERGY (TJ)"]] + + ngd = ( + ngd.groupby(["YEAR", "EXPORTER", "IMPORTER", "MESSAGE COMMODITY"])[ + "ENERGY (TJ)" + ] + .sum() + .reset_index() + ) + + return ngd + + +# Check against IEA balances +def import_iea_balances( + project_name: str | None = None, config_name: str | None = None +): + """ + Import IEA balances and save as CSV. + + Args: + project_name: Name of project (e.g., 'newpathways') + config_name: Name of config file + """ + data_paths = setup_datapath(project_name=project_name, config_name=config_name) + + ieadf1 = pd.read_csv( + os.path.join(data_paths["iea_web"], "EARLYBIG1.txt"), sep=r"\s+", header=None + ) + ieadf2 = pd.read_csv( + os.path.join(data_paths["iea_web"], "EARLYBIG2.txt"), sep=r"\s+", header=None + ) + + ieadf = pd.concat([ieadf1, ieadf2]) + ieadf.columns = [ + "region", + "fuel", + "year", + "flow", + "unit", + "value", + "statisticalerror", + ] + + iea_out = pd.DataFrame() + for t in ["EXPORTS", "IMPORTS"]: + tdf = ieadf[ieadf["flow"] == t].copy() + tdf = tdf[tdf["unit"] == "TJ"] + + tdf = tdf[["region", "fuel", "year", "flow", "unit", "value"]] + tdf = tdf.rename( + columns={ + "region": "REGION", + "fuel": "IEA-WEB COMMODITY", + "year": "YEAR", + "flow": "FLOW", + "unit": "IEA-WEB UNIT", + "value": "IEA-WEB VALUE", + } + ) + iea_out = pd.concat([iea_out, tdf]) + + iea_out.to_csv(os.path.join(data_paths["iea_web"], "WEB_TRADEFLOWS.csv")) + + +def check_iea_balances( + indf, project_name: str | None = None, config_name: str | None = None +): + """ + Check against IEA balances. + + Args: + indf: Input dataframe + project_name: Name of project (e.g., 'newpathways') + config_name: Name of config file + """ + data_paths = setup_datapath(project_name=project_name, config_name=config_name) + + iea = pd.read_csv(os.path.join(data_paths["iea_web"], "WEB_TRADEFLOWS.csv")) + ieacw = pd.read_csv(os.path.join(data_paths["iea_web"], "country_crosswalk.csv")) + iea = iea.merge(ieacw, left_on="REGION", right_on="REGION", how="left") + iea["IEA-WEB VALUE"] = np.where( + iea["FLOW"] == "EXPORTS", iea["IEA-WEB VALUE"] * -1, iea["IEA-WEB VALUE"] + ) + + # LNG and pipe gas are directly from IEA + indf = indf[~indf["MESSAGE COMMODITY"].isin(["gas_piped", "LNG_shipped"])].copy() + + dict_dir = package_data_path("bilateralize", "commodity_codes.yaml") + with open(dict_dir, "r", encoding="utf8") as f: + commodity_codes = yaml.safe_load(f) + + iea["COMMODITY"] = "" + indf["COMMODITY"] = "" + for c in commodity_codes.keys(): + iea["COMMODITY"] = np.where( + iea["IEA-WEB COMMODITY"].isin(commodity_codes[c]["IEA-WEB"]), + c, + iea["COMMODITY"], + ) + indf["COMMODITY"] = np.where( + indf["MESSAGE COMMODITY"] == commodity_codes[c]["MESSAGE Commodity"], + c, + indf["COMMODITY"], + ) + + exports = ( + indf.groupby(["YEAR", "EXPORTER", "COMMODITY"])["ENERGY (TJ)"] + .sum() + .reset_index() + ) + imports = ( + indf.groupby(["YEAR", "IMPORTER", "COMMODITY"])["ENERGY (TJ)"] + .sum() + .reset_index() + ) + + exports = exports.merge( + iea[iea["FLOW"] == "EXPORTS"][ + ["ISO", "COMMODITY", "YEAR", "IEA-WEB UNIT", "IEA-WEB VALUE"] + ], + left_on=["YEAR", "EXPORTER", "COMMODITY"], + right_on=["YEAR", "ISO", "COMMODITY"], + how="left", + ) + imports = imports.merge( + iea[iea["FLOW"] == "IMPORTS"][ + ["ISO", "COMMODITY", "YEAR", "IEA-WEB UNIT", "IEA-WEB VALUE"] + ], + left_on=["YEAR", "IMPORTER", "COMMODITY"], + right_on=["YEAR", "ISO", "COMMODITY"], + how="left", + ) + + exports["DIFFERENCE"] = ( + exports["ENERGY (TJ)"] - exports["IEA-WEB VALUE"] + ) / exports["IEA-WEB VALUE"] + imports["DIFFERENCE"] = ( + imports["ENERGY (TJ)"] - imports["IEA-WEB VALUE"] + ) / imports["IEA-WEB VALUE"] + + exports.to_csv(os.path.join(data_paths["iea_diag"], "iea_calibration_exports.csv")) + imports.to_csv(os.path.join(data_paths["iea_diag"], "iea_calibration_imports.csv")) + + +# Aggregate UN Comtrade data to MESSAGE regions +def reformat_to_parameter( + indf, + message_regions, + parameter_name, + project_name=None, + config_name=None, + exports_only=False, +): + """ + Aggregate UN Comtrade data to MESSAGE regions and + set up historical activity parameter dataframe. + + Args: + indf: Input dataframe + message_regions: Regional resolution + parameter_name: Name of parameter + project_name: Name of project (e.g., 'newpathways') + config_name: Name of config file + exports_only: If True, only include exports + """ + dict_dir = package_data_path("node", message_regions + ".yaml") + with open(dict_dir, "r", encoding="utf8") as f: + dict_message_regions = yaml.safe_load(f) + region_list = [i for i in list(dict_message_regions.keys()) if i != "World"] + + indf["EXPORTER REGION"] = "" + indf["IMPORTER REGION"] = "" + for t in ["EXPORTER", "IMPORTER"]: + for r in region_list: + indf[t + " REGION"] = np.where( + indf[t].isin(dict_message_regions[r]["child"]), r, indf[t + " REGION"] + ) + + # Collapse to regional level + if parameter_name in ["historical_activity"]: + indf = ( + indf.groupby( + ["YEAR", "EXPORTER REGION", "IMPORTER REGION", "MESSAGE COMMODITY"] + )[["ENERGY (GWa)"]] + .sum() + .reset_index() + ) + metric_name = "ENERGY (GWa)" + elif parameter_name in ["var_cost", "inv_cost", "fix_cost"]: + indf = ( + indf.groupby(["EXPORTER REGION", "IMPORTER REGION", "MESSAGE COMMODITY"])[ + ["ENERGY (GWa)", "VALUE (MUSD)"] + ] + .sum() + .reset_index() + ) + indf["PRICE (MUSD/GWa)"] = indf["VALUE (MUSD)"] / indf["ENERGY (GWa)"] + indf["YEAR"] = "broadcast" + metric_name = "PRICE (MUSD/GWa)" + + indf = indf[(indf["EXPORTER REGION"] != "") & (indf["IMPORTER REGION"] != "")] + indf = indf[indf["EXPORTER REGION"] != indf["IMPORTER REGION"]] + + # Add MESSAGE columns for exports + exdf = message_ix.make_df( + parameter_name, + node_loc=indf["EXPORTER REGION"], + technology=indf["MESSAGE COMMODITY"] + + "_exp_" + + indf["IMPORTER REGION"].str.replace(message_regions + "_", "").str.lower(), + year_act=indf["YEAR"], + year_vtg=indf["YEAR"], + value=indf[metric_name], + mode="M1", + time="year", + ) + outdf = exdf.copy() + + # Add MESSAGE columns for imports + if not exports_only: + imdf = message_ix.make_df( + parameter_name, + node_loc=indf["IMPORTER REGION"], + technology=indf["MESSAGE COMMODITY"] + "_imp", + year_act=indf["YEAR"], + year_vtg=indf["YEAR"], + value=indf[metric_name], + mode="M1", + time="year", + ) + + outdf = pd.concat([outdf, imdf]) + + return outdf + + +# Run all for historical activity +def build_historical_activity( + message_regions="R12", + project_name: str | None = None, + config_name: str | None = None, + reimport_IEA=False, + reimport_BACI=False, +): + """ + Build historical activity parameter dataframe. + + Args: + message_regions: Regional resolution + project_name: Name of project (e.g., 'newpathways') + config_name: Name of config file + reimport_IEA: If True, reimport IEA data + reimport_BACI: If True, reimport BACI data + """ + if reimport_IEA: + generate_cfdict( + message_regions=message_regions, + project_name=project_name, + config_name=config_name, + ) + import_iea_balances(project_name=project_name, config_name=config_name) + if reimport_BACI: + import_uncomtrade(project_name=project_name, config_name=config_name) + + bacidf = convert_trade( + message_regions=message_regions, + project_name=project_name, + config_name=config_name, + ) + bacidf = bacidf[bacidf["MESSAGE COMMODITY"] != "lng"] # Get LNG from IEA instead + bacidf["MESSAGE COMMODITY"] = bacidf["MESSAGE COMMODITY"] + "_shipped" + + ngdf = import_iea_gas(project_name=project_name, config_name=config_name) + + # Calibrate gas to IEA WEB data + data_paths = setup_datapath(project_name=project_name, config_name=config_name) + web = pd.read_csv(os.path.join(data_paths["iea_web"], "WEB_TRADEFLOWS.csv")) + ieacw = pd.read_csv(os.path.join(data_paths["iea_web"], "country_crosswalk.csv")) + web = web.merge(ieacw, left_on="REGION", right_on="REGION", how="left") + web["IEA-WEB VALUE"] = np.where( + web["FLOW"] == "EXPORTS", web["IEA-WEB VALUE"] * -1, web["IEA-WEB VALUE"] + ) + web = web[web["IEA-WEB COMMODITY"] == "NATURAL_GAS"] + + web_tot = web[web["FLOW"] == "EXPORTS"][["YEAR", "ISO", "IEA-WEB VALUE"]] + web_tot = web_tot.groupby(["YEAR", "ISO"])["IEA-WEB VALUE"].sum().reset_index() + web_tot = web_tot.rename(columns={"ISO": "EXPORTER", "IEA-WEB VALUE": "WEB TOTAL"}) + ngdf_tot = ngdf.groupby(["YEAR", "EXPORTER"])["ENERGY (TJ)"].sum().reset_index() + ngdf_tot = ngdf_tot.rename( + columns={"EXPORTER": "EXPORTER", "ENERGY (TJ)": "NGDF TOTAL"} + ) + ngdf_tot = ngdf_tot.merge( + web_tot, + left_on=["YEAR", "EXPORTER"], + right_on=["YEAR", "EXPORTER"], + how="outer", + ) + + ngdf_tot["CALIBRATION FACTOR"] = ngdf_tot["WEB TOTAL"] / ngdf_tot["NGDF TOTAL"] + ngdf_tot["CALIBRATION FACTOR"] = np.where( + ngdf_tot["CALIBRATION FACTOR"].isnull(), 1, ngdf_tot["CALIBRATION FACTOR"] + ) + ngdf_tot["CALIBRATION FACTOR"] = np.where( + ngdf_tot["CALIBRATION FACTOR"] < 1, 1, ngdf_tot["CALIBRATION FACTOR"] + ) + ngdf_tot = ngdf_tot[["YEAR", "EXPORTER", "CALIBRATION FACTOR"]] + + ngdf = ngdf.merge( + ngdf_tot, + left_on=["YEAR", "EXPORTER"], + right_on=["YEAR", "EXPORTER"], + how="left", + ) + ngdf["ENERGY (TJ)"] = ngdf["ENERGY (TJ)"] * ngdf["CALIBRATION FACTOR"] + ngdf = ngdf.drop(columns=["CALIBRATION FACTOR"]) + + tradedf = bacidf.merge( + ngdf, + left_on=["YEAR", "EXPORTER", "IMPORTER", "MESSAGE COMMODITY"], + right_on=["YEAR", "EXPORTER", "IMPORTER", "MESSAGE COMMODITY"], + how="outer", + ) + tradedf["ENERGY (TJ)"] = tradedf["ENERGY (TJ)_x"] + tradedf["ENERGY (TJ)"] = np.where( + tradedf["MESSAGE COMMODITY"].isin(["LNG_shipped", "gas_piped"]), + tradedf["ENERGY (TJ)_y"], + tradedf["ENERGY (TJ)"], + ) + tradedf["ENERGY (TJ)"] = tradedf["ENERGY (TJ)"].astype(float) + tradedf = tradedf[ + ["YEAR", "EXPORTER", "IMPORTER", "HS", "MESSAGE COMMODITY", "ENERGY (TJ)"] + ].reset_index() + + check_iea_balances(indf=tradedf, project_name=project_name, config_name=config_name) + + tradedf["ENERGY (GWa)"] = tradedf["ENERGY (TJ)"] * (3.1712 * 1e-5) # TJ to GWa + + outdf = reformat_to_parameter( + indf=tradedf, + message_regions=message_regions, + parameter_name="historical_activity", + project_name=project_name, + config_name=config_name, + ) + outdf["unit"] = "GWa" + + return outdf.drop_duplicates() + + +# Calculate historical new capacity based on activity +def build_hist_new_capacity_trade( + message_regions="R12", + project_name: str | None = None, + config_name: str | None = None, +): + """ + Build historical new capacity based on activity. + + Args: + message_regions: Regional resolution + project_name: Name of project (e.g., 'newpathways') + config_name: Name of config file + """ + indf = build_historical_activity( + message_regions=message_regions, + project_name=project_name, + config_name=config_name, + ) + + basedf = pd.DataFrame() + for y in list(range(2000, 2030, 5)): + ydf = ( + indf[["node_loc", "technology", "mode", "time", "unit"]] + .drop_duplicates() + .copy() + ) + ydf["year_act"] = y + basedf = pd.concat([basedf, ydf]) + + df = basedf.merge( + indf, + left_on=["node_loc", "technology", "mode", "time", "unit", "year_act"], + right_on=["node_loc", "technology", "mode", "time", "unit", "year_act"], + how="outer", + ) + + df["year_5"] = round(df["year_act"] / 5, 0) * 5 # Get closest 5 + df["year_act"] = np.where(df["year_act"] == 2023, 2025, df["year_act"]) + df = df[df["year_act"] == df["year_5"]] + df["value"] = np.where(df["value"].isnull(), 0, df["value"]) + + df = df.sort_values(by=["node_loc", "technology", "mode", "time", "year_act"]) + df["value"] = df["value"].diff() + + df = df[df["year_act"] > 2000] # Start at 2000, this ensures diff starts correctly + df["value"] = np.where(df["value"] < 0, 0, df["value"]) + df["year_vtg"] = df["year_act"].astype(int) + + df = df[["node_loc", "technology", "year_vtg", "value", "unit"]].drop_duplicates() + + return df + + +# Run all for price +def build_historical_price( + message_regions="R12", + project_name: str | None = None, + config_name: str | None = None, +): + """ + Build historical price parameter dataframe. + + Args: + message_regions: Regional resolution + project_name: Name of project (e.g., 'newpathways') + config_name: Name of config file + """ + if reimport_BACI: + import_uncomtrade(project_name=project_name, config_name=config_name) + + bacidf = convert_trade( + message_regions=message_regions, + project_name=project_name, + config_name=config_name, + ) + bacidf["MESSAGE COMMODITY"] = np.where( + bacidf["MESSAGE COMMODITY"] == "lng", "LNG", bacidf["MESSAGE COMMODITY"] + ) + bacidf["MESSAGE COMMODITY"] = bacidf["MESSAGE COMMODITY"] + "_shipped" + bacidf = bacidf[bacidf["YEAR"] > 2020] + + bacidf["ENERGY (GWa)"] = bacidf["ENERGY (TJ)"] * (3.1712 * 1e-5) # TJ to GWa + bacidf["VALUE (MUSD)"] = bacidf["VALUE (1000USD)"] * 1e-3 + bacidf["PRICE (MUSD/GWa)"] = bacidf["VALUE (MUSD)"] / bacidf["ENERGY (GWa)"] + + bacidf = bacidf[bacidf["ENERGY (TJ)"] > 0.5] # Keep linkages >0.5TJ + + bacidf = ( + bacidf.groupby(["EXPORTER", "IMPORTER", "MESSAGE COMMODITY"])[ + ["ENERGY (GWa)", "VALUE (MUSD)"] + ] + .sum() + .reset_index() + ) + bacidf["YEAR"] = "broadcast" + + outdf = reformat_to_parameter( + indf=bacidf, + message_regions=message_regions, + parameter_name="var_cost", + project_name=project_name, + config_name=config_name, + exports_only=True, + ) + outdf["unit"] = "USD/GWa" + + outdf["value"] = outdf["value"] * 0.50 # TODO: Fix this deflator (2024-2005?) + outdf["value"] = round(outdf["value"], 0) + + return outdf + + +# Build for historical new capacity of a given maritime shipment (e.g., LNG tanker) +def build_hist_new_capacity_flow( + infile: str, + ship_type: str, + message_regions: str = "R12", + project_name: str | None = None, + config_name: str | None = None, + annual_mileage=100000, +): + """ + Build historical new capacity of a given maritime shipment (e.g., LNG tanker). + + Args: + infile: Name of GISIS input file + ship_type: Ship type (e.g., 'LNG_tanker_loil') + message_regions: Regional resolution + project_name: Name of project (e.g., 'newpathways') + config_name: Name of config file + annual_mileage: Average annual mileage of ship in km + """ + # Regions + dict_dir = package_data_path("node", message_regions + ".yaml") + with open(dict_dir, "r", encoding="utf8") as f: + dict_message_regions = yaml.safe_load(f) + region_list = [i for i in list(dict_message_regions.keys()) if i != "World"] + + # IMO data + data_paths = setup_datapath(project_name=project_name, config_name=config_name) + imodf = pd.read_csv(os.path.join(data_paths["imo"], "GISIS", infile)) + + # Get MESSAGE regions + imodf[message_regions] = "" + for r in region_list: + imodf[message_regions] = np.where( + imodf["Flag ISO3"].isin(dict_message_regions[r]["child"]), + r, + imodf[message_regions], + ) + + # Calculate capacity + if imodf["Gross Tonnage"].dtype in ["O", "str"]: + imodf["Gross Tonnage"] = imodf["Gross Tonnage"].str.replace(",", "").astype(int) + imodf["Capacity (Mt-km)"] = (imodf["Gross Tonnage"] / 1e6) * annual_mileage + + # Collapse + imodf["Year of Build (5)"] = round(imodf["Year of Build"].astype(float) / 5) * 5 + imodf = ( + imodf.groupby([message_regions, "Year of Build (5)"])[["Capacity (Mt-km)"]] + .sum() + .reset_index() + ) + imodf["Capacity (Mt-km) (Annualized)"] = round(imodf["Capacity (Mt-km)"] / 5, 0) + + # Parameterize + imodf = imodf.rename( + columns={ + message_regions: "node_loc", + "Year of Build (5)": "year_vtg", + "Capacity (Mt-km) (Annualized)": "value", + } + ) + imodf["technology"] = ship_type + imodf["unit"] = "Mt-km" + imodf["year_vtg"] = imodf["year_vtg"].astype(int) + + imodf = imodf[["node_loc", "technology", "year_vtg", "value", "unit"]] + imodf = imodf[imodf["node_loc"] != ""] + + return imodf diff --git a/message_ix_models/tools/bilateralize/load_and_solve.py b/message_ix_models/tools/bilateralize/load_and_solve.py new file mode 100644 index 0000000000..cdd4ff42b9 --- /dev/null +++ b/message_ix_models/tools/bilateralize/load_and_solve.py @@ -0,0 +1,433 @@ +# -*- coding: utf-8 -*- +""" +Update MESSAGEix scenario(s) with bilateralized dictionary + +This script is the third step in implementing the bilateralize tool. +It updates a specified MESSAGEix scenario with the bilateralized dictionary. +It then has options to solve the scenario within the ixmp database +or save as a GDX data file for direct solve in GAMS. +""" + +# Import packages +import logging +import os +from pathlib import Path + +import ixmp +import message_ix +import pandas as pd + +from message_ix_models.tools.bilateralize.utils import get_logger, load_config + + +# %% Remove existing trade technologies +def remove_trade_tech(scen: message_ix.Scenario, log, config_tec: dict, tec: str): + """ + Remove existing trade technologies + """ + base_tec_name = tec.replace("_shipped", "") + base_tec_name = base_tec_name.replace("_piped", "") + + base_tec = [ + config_tec[tec][tec + "_trade"]["trade_commodity"] + "_exp", + config_tec[tec][tec + "_trade"]["trade_commodity"] + "_imp", + base_tec_name + "_exp", + base_tec_name + "_imp", + ] + base_tec = base_tec + [ + i + for i in scen.set("technology") + if config_tec[tec][tec + "_trade"]["trade_commodity"] + "_exp_" in i + ] + if "crudeoil" in tec: + base_tec = base_tec + ["oil_exp", "oil_imp"] # for crude + + base_tec = list(set(base_tec)) + with scen.transact("Remove base trade technologies for " + tec): + for t in base_tec: + if t in list(scen.set("technology")): + log.info("Removing base technology..." + t) + scen.remove_set("technology", t) + + +# %% Add sets for trade technologies +def add_trade_sets(scen: message_ix.Scenario, log, trade_dict: dict, tec: str): + """ + Add sets for trade technologies + """ + new_sets = dict() + for s in ["technology", "level", "commodity", "mode"]: + setlist = set( + list(trade_dict[tec]["trade"]["input"][s].unique()) + + list(trade_dict[tec]["trade"]["output"][s].unique()) + ) + + if "input" in trade_dict[tec]["flow"].keys(): + add_list = set( + list(trade_dict[tec]["flow"]["input"][s].unique()) + + list(trade_dict[tec]["flow"]["output"][s].unique()) + ) + setlist = setlist.union(add_list) + setlist_out = list(setlist) + + new_sets[s] = setlist_out + + with scen.transact("Add new sets for " + tec): + for s in ["technology", "level", "commodity", "mode"]: + base_set = list(scen.set(s)) + for i in new_sets[s]: + if i not in base_set: + log.info("Adding set: " + s + "..." + i) + scen.add_set(s, i) + else: + pass + + +# %% Add parameters for bilateralized trade +def add_trade_parameters(scen: message_ix.Scenario, log, trade_dict: dict, tec: str): + """ + Add parameters for bilateralized trade + """ + new_parameter_list = list( + set( + [ + i + for i in list(trade_dict[tec]["trade"].keys()) + + list(trade_dict[tec]["flow"].keys()) + if "relation_" not in i + ] + ) + ) + + with scen.transact("Add new parameters for " + tec): + for p in new_parameter_list: + log.info("Adding parameter for " + tec + ": " + p) + pardf = pd.DataFrame() + if trade_dict[tec]["trade"].get(p) is not None: + log.info("... parameter added for trade technology") + pardf = pd.concat([pardf, trade_dict[tec]["trade"][p]]) + if trade_dict[tec]["flow"].get(p) is not None: + log.info("... parameter added for flow technology") + pardf = pd.concat([pardf, trade_dict[tec]["flow"][p]]) + pardf = pardf[pardf["value"].notnull()] + scen.add_par(p, pardf) + + +# %% Update relation parameters +def update_relation_parameters( + scen: message_ix.Scenario, log, trade_dict: dict, tec: str +): + """ + Update relation parameters + """ + rel_parameter_list = list( + set( + [ + i + for i in list(trade_dict[tec]["trade"].keys()) + + list(trade_dict[tec]["flow"].keys()) + if "relation_" in i + ] + ) + ) + + if len(rel_parameter_list) > 0: + for rel_par in ["relation_activity", "relation_upper", "relation_lower"]: + rel_par_df = pd.DataFrame() + + rel_par_list = list( + set( + [ + i + for i in list(trade_dict[tec]["trade"].keys()) + + list(trade_dict[tec]["flow"].keys()) + if rel_par in i + ] + ) + ) + + for r in rel_par_list: + if r in list(trade_dict[tec]["trade"].keys()): + rel_par_df = pd.concat([rel_par_df, trade_dict[tec]["trade"][r]]) + if r in list(trade_dict[tec]["flow"].keys()): + rel_par_df = pd.concat([rel_par_df, trade_dict[tec]["flow"][r]]) + + if rel_par == "relation_activity": + with scen.transact("Adding new relation sets"): + new_relations = list(rel_par_df["relation"].unique()) + for nr in new_relations: + if nr not in scen.set("relation").unique(): + scen.add_set("relation", nr) + if len(rel_par_df) > 0: + with scen.transact("Add " + rel_par + " for " + tec): + log.info("Adding " + rel_par + " for " + tec) + rel_par_df = rel_par_df[rel_par_df["value"].notnull()] + scen.add_par(rel_par, rel_par_df) + + +# %% Update bunker fuels +def update_bunker_fuels(scen: message_ix.Scenario, tec: str, log, config_tec: dict): + """ + Update bunker fuels + """ + bunker_tec = config_tec[tec][tec + "_trade"]["bunker_technology"] + if bunker_tec is not None: + for btec in bunker_tec.keys(): + bunkerdf_in = scen.par("input", filters={"technology": bunker_tec[btec]}) + bunkerdf_out = bunkerdf_in.copy() + bunkerdf_out["level"] = "bunker" + with scen.transact("Update bunker fuel for" + tec): + log.info("Updating bunker level for " + tec) + scen.remove_par("input", bunkerdf_in) + scen.add_par("input", bunkerdf_out) + + +# %% Update additional parameters (separate from bilateralization) +def update_additional_parameters( + scen: message_ix.Scenario, extra_parameter_updates: dict | None = None +): + """ + Update additional parameters (separate from bilateralization) + """ + if extra_parameter_updates is not None: + for par in extra_parameter_updates.keys(): + with scen.transact("Update additional parameter: " + par): + new_df = extra_parameter_updates[par] + base_df = scen.par(par) + + if "value" in new_df.columns: + rem_df = new_df[[c for c in new_df.columns if c != "value"]] + base_df = base_df.merge( + rem_df, + left_on=list(rem_df.columns), + right_on=list(rem_df.columns), + how="inner", + ) + add_df = base_df.copy().drop(["value"], axis=1) + add_df = add_df.merge( + new_df, + left_on=list(rem_df.columns), + right_on=list(rem_df.columns), + how="left", + ) + + if "multiplier" in new_df.columns: + col_list = [c for c in new_df.columns if c != "multiplier"] + base_df = base_df.merge( + new_df, left_on=col_list, right_on=col_list, how="inner" + ) + add_df = base_df.copy() + base_df = base_df.drop(["multiplier"], axis=1) + add_df["value"] = add_df["value"] * add_df["multiplier"] + add_df = add_df.drop(["multiplier"], axis=1) + + scen.remove_par(par, base_df) + scen.add_par(par, add_df) + + +# %% Remove PAO constraints on MESSAGEix-GLOBIOM +def remove_pao_coal_constraint( + scen: message_ix.Scenario, log, MESSAGEix_GLOBIOM: bool = True +): + """ + Remove PAO coal and gas constraints on MESSAGEix-GLOBIOM + """ + if MESSAGEix_GLOBIOM: + with scen.transact("Remove PAO coal and gas constraints on primary energy"): + for rel in ["domestic_coal", "domestic_gas"]: + log.info("Removing constraints on PAO primary energy: " + rel) + relact_df = scen.par("relation_activity", filters={"relation": rel}) + relupp_df = scen.par("relation_upper", filters={"relation": rel}) + scen.remove_par("relation_activity", relact_df) + scen.remove_par("relation_upper", relupp_df) + + +# %% Write just the GDX files +def save_to_gdx(mp: ixmp.Platform, scenario, output_path: str): + """ + Save the scenario to a GDX file. + + Args: + mp: ixmp platform + scenario: Scenario name + output_path: Path to save the GDX file + """ + from ixmp.backend import ItemType + + mp._backend.write_file( + Path(output_path), + ItemType.SET | ItemType.PAR, + filters={"scenario": scenario}, + ) + # %% Solve or save scenario + + +def solve_or_save( + mp: ixmp.Platform, + scen: message_ix.Scenario, + solve: bool = False, + to_gdx: bool = False, + gdx_location: str | None = None, +): + """ + Solve or save scenario. + """ + if to_gdx and not solve and gdx_location is not None: + save_to_gdx( + mp=mp, + scenario=scen, + output_path=os.path.join( + gdx_location, f"MsgData_{scen.model}_{scen.scenario}.gdx" + ), + ) + + if solve: + solver = "MESSAGE" + scen.solve(solver, solve_options=dict(lpmethod=4)) + + print("Unlock run ID of the scenario") + # runid = scen.run_id() + # mp._backend.jobj.unlockRunid(runid) + + +def load_and_clone( + mp: ixmp.Platform, + log: logging.Logger, + config_base: dict, + start_scen: str | None = None, + start_model: str | None = None, + target_scen: str | None = None, + target_model: str | None = None, +) -> message_ix.Scenario: + """Load and clone scenario. + Args: + mp: ixmp platform + log: Logger + project_name: Name of project + config_name: Name of config file + start_scen: Name of scenario to start from + start_model: Name of model to start from + target_scen: Name of scenario to target + target_model: Name of model to target + """ + # Load config + if start_model is None: + start_model = config_base.get("scenario", {}).get("start_model") + if start_scen is None: + start_scen = config_base.get("scenario", {}).get("start_scen") + + base = message_ix.Scenario(mp, model=start_model, scenario=start_scen) + log.info(f"Loaded scenario: {start_model}/{start_scen}") + + # Clone scenario + if target_model is None: + target_model = config_base.get("scenario", {}).get("target_model", []) + if target_scen is None: + target_scen = config_base.get("scenario", {}).get("target_scen") + + scen = base.clone(target_model, target_scen, keep_solution=False) + scen.set_as_default() + + log.info("Scenario loaded and cloned.") + + return scen + + +# %% Clone and update scenario +def load_and_solve( + trade_dict, + solve=False, + to_gdx=False, + project_name: str | None = None, + config_name: str | None = None, + start_scen: str | None = None, + start_model: str | None = None, + target_scen: str | None = None, + target_model: str | None = None, + scenario: message_ix.Scenario | None = None, + extra_parameter_updates: dict | None = None, + gdx_location: str | None = None, + MESSAGEix_GLOBIOM: bool = True, +): + """ + Clone and update scenario. + + Args: + trade_dict: Dictionary of parameter dataframes + Optional Args: + solve: If True, solve scenario + to_gdx: If True, save scenario to a GDX file + project_name: Name of project (message_ix_models/project/[THIS]) + config_name: Name of config file. + If None, uses default config from data/bilateralize/config_default.yaml + start_scen: Name of scenario to start from + start_model: Name of model to start from + start_model_name: Name of model to start from + target_scen: Name of scenario to target + target_model: Name of model to target + target_model_name: Name of model to target + scenario: Scenario to update (if None, will clone from project yaml) + additional_parameter_updates: Dictionary of additional parameter updates + gdx_location: Location to save GDX file + remove_pao_coal_constraint: Remove PAO coal and gas constraints + """ + # Load config + config_base, config_path, config_tec = load_config( + project_name, config_name, load_tec_config=True + ) + + log = get_logger(name="load_and_solve") + log.info("Loading and solving scenario") + + # Load the scenario + mp = ixmp.Platform() + + if scenario is None: + scen = load_and_clone( + mp=mp, + log=log, + config_base=config_base, + start_scen=start_scen, + start_model=start_model, + target_scen=target_scen, + target_model=target_model, + ) + else: + log.info(f"Using existing scenario: {scenario.model}/{scenario.scenario}") + scen = scenario + + # Add sets and parameters for each covered technology + covered_tec = config_base.get("covered_trade_technologies") + + for tec in covered_tec: + # Remove existing technologies related to trade + remove_trade_tech(scen=scen, log=log, config_tec=config_tec, tec=tec) + + # Add to sets: technology, level, commodity, mode + add_trade_sets(scen=scen, log=log, trade_dict=trade_dict, tec=tec) + + # Add parameters + add_trade_parameters(scen=scen, log=log, trade_dict=trade_dict, tec=tec) + + # Relation activity, upper, and lower + update_relation_parameters(scen=scen, log=log, trade_dict=trade_dict, tec=tec) + + # Update bunker fuels + update_bunker_fuels(scen=scen, tec=tec, log=log, config_tec=config_tec) + + # Update additional parameters + update_additional_parameters( + scen=scen, extra_parameter_updates=extra_parameter_updates + ) + + # Remove PAO coal and gas constraints on MESSAGEix-GLOBIOM + remove_pao_coal_constraint(scen=scen, log=log, MESSAGEix_GLOBIOM=MESSAGEix_GLOBIOM) + + # Solve or save scenario + solve_or_save( + mp=mp, + scen=scen, + solve=solve, + to_gdx=to_gdx, + gdx_location=gdx_location, + ) diff --git a/message_ix_models/tools/bilateralize/mariteam_calibration.py b/message_ix_models/tools/bilateralize/mariteam_calibration.py new file mode 100644 index 0000000000..08f376562c --- /dev/null +++ b/message_ix_models/tools/bilateralize/mariteam_calibration.py @@ -0,0 +1,238 @@ +# -*- coding: utf-8 -*- +""" +Calibration with MariTEAM output +""" + +# Import packages +import os + +import numpy as np +import pandas as pd + +from message_ix_models.tools.bilateralize.utils import load_config +from message_ix_models.util import package_data_path + + +def calibrate_mariteam( + covered_tec, + message_regions, + mtdict={ + "LNG_shipped": { + "astd_ship_type": "Gas tankers", + "flow_technology": ["LNG_tanker_LNG", "LNG_tanker_loil"], + }, + "crudeoil_shipped": { + "astd_ship_type": "Crude oil tankers", + "flow_technology": ["crudeoil_tanker_loil"], + }, + "coal_shipped": { + "astd_ship_type": "Bulk carriers", + "flow_technology": ["energy_bulk_carrier_loil"], + }, + "eth_shipped": { + "astd_ship_type": "Oil product tankers", + "flow_technology": ["oil_tanker_eth", "oil_tanker_loil"], + }, + "foil_shipped": { + "astd_ship_type": "Oil product tankers", + "flow_technology": ["oil_tanker_loil", "oil_tanker_foil"], + }, + "loil_shipped": { + "astd_ship_type": "Oil product tankers", + "flow_technology": ["oil_tanker_loil"], + }, + "biomass_shipped": { + "astd_ship_type": "Bulk carriers", + "flow_technology": ["energy_bulk_carrier_loil"], + }, + "lh2_shipped": { + "astd_ship_type": "Chemical tankers", + "flow_technology": ["lh2_tanker_lh2", "lh2_tanker_loil"], + }, + }, + mt_output="MariTEAM_output_2025-07-21.csv", + project_name: str | None = None, + config_name: str | None = None, +): + # Data paths + config, config_path = load_config( + project_name=project_name, config_name=config_name + ) + p_drive = config["p_drive_location"] + data_path = os.path.join(p_drive, "MESSAGE_trade") + mt_path = os.path.join(data_path, "MariTEAM") + out_path = os.path.join( + os.path.dirname(package_data_path("bilateralize")), "bilateralize" + ) + + # Import MariTEAM outputs + mtdf = pd.read_csv(os.path.join(mt_path, mt_output)) + mtdf = mtdf[ + mtdf[message_regions + "_origin"] != mtdf[message_regions + "_destination"] + ].copy() # no intraregional trade + + for tec in [i for i in covered_tec if "shipped" in i]: + # If a non-energy commodity, add to bulk carriers in mtdict + if tec.replace("_shipped", "") not in [ + "biomass", + "coal", + "crudeoil", + "eth", + "foil", + "lh2", + "loil", + "LNG", + "meth", + ]: + mtdict_add = { + tec: { + "astd_ship_type": "Bulk carriers", + "flow_technology": ["energy_bulk_carrier_loil"], + } + } + mtdict.update(mtdict_add) + + for flow_fuel in mtdict[tec]["flow_technology"]: + basedf = mtdf[ + mtdf["astd_ship_type"] == mtdict[tec]["astd_ship_type"] + ].copy() + basedf["node_loc"] = basedf[message_regions + "_origin"] + basedf["technology"] = flow_fuel + + # Fuel consumption (input) + mt_input = basedf.copy() + mt_input["mt_value"] = ( + mt_input["intensity_MJ_tonne"] / mt_input["distance_km_sum"] + ) # MJ/t-km + mt_input["mt_value"] = mt_input["mt_value"] * 3.17e-11 # GWa/t-km + mt_input["mt_value"] = mt_input["mt_value"] * 1e6 # GWa/Mt-km + mt_input["unit"] = "GWa" # denominator assumed in output + mt_input = ( + mt_input.groupby(["node_loc", "technology", "unit"])["mt_value"] + .sum() + .reset_index() + ) + + regavg = ( + basedf.groupby(["node_loc"])[ + ["energy_mj_sum", "dwt", "distance_km_sum"] + ] + .sum() + .reset_index() + ) + regavg["mt_value_reg"] = regavg["energy_mj_sum"] / ( + regavg["dwt"] * regavg["distance_km_sum"] + ) # MJ/t-km + regavg = regavg[["node_loc", "mt_value_reg"]] + + inputdf = pd.read_csv( + os.path.join( + out_path, tec, "edit_files", "flow_technology", "input.csv" + ) + ) + inputdf = inputdf.merge( + mt_input, + left_on=["node_loc", "technology", "unit"], + right_on=["node_loc", "technology", "unit"], + how="left", + ) + inputdf = inputdf.merge( + regavg, left_on=["node_loc"], right_on=["node_loc"], how="left" + ) + + inputdf["value"] = np.where( + inputdf["mt_value"] > 0, inputdf["mt_value"], inputdf["value"] + ) + inputdf["value"] = np.where( + (inputdf["value"].isnull()) + & (inputdf["technology"].str.contains(flow_fuel)), + inputdf["mt_value_reg"], + inputdf["value"], + ) + + inputdf = inputdf[ + [ + "node_origin", + "node_loc", + "technology", + "year_vtg", + "year_act", + "mode", + "commodity", + "level", + "value", + "time", + "time_origin", + "unit", + ] + ] + inputdf.to_csv( + os.path.join( + out_path, tec, "edit_files", "flow_technology", "input.csv" + ), + index=False, + ) + + if not os.path.isfile( + os.path.join( + out_path, tec, "bare_files", "flow_technology", "input.csv" + ) + ): + inputdf.to_csv( + os.path.join( + out_path, tec, "bare_files", "flow_technology", "input.csv" + ), + index=False, + ) + + # Historical activity + histdf = pd.DataFrame() + for flow_fuel in mtdict[tec]["flow_technology"]: + basedf = mtdf[ + mtdf["astd_ship_type"] == mtdict[tec]["astd_ship_type"] + ].copy() + basedf["node_loc"] = basedf[message_regions + "_origin"] + basedf["technology"] = flow_fuel + basedf["value"] = basedf["distance_km_sum"] * basedf["dwt"] # t-km + basedf["value"] = basedf["value"] / 1e6 # Mt-km + basedf["unit"] = "Mt-km" + basedf["year_act"] = 2025 # last historical year + basedf["mode"] = ( + basedf["R12_origin"].str.replace(message_regions + "_", "") + + "-" + + basedf["R12_destination"].str.replace(message_regions + "_", "") + ) + basedf["time"] = "year" + basedf = basedf[ + ["node_loc", "technology", "year_act", "value", "unit", "mode", "time"] + ].copy() + + shipping_fuel_dict = config["shipping_fuels"] + + if "LNG_tanker" in flow_fuel: + basedf["value"] *= shipping_fuel_dict["LNG_tanker"][flow_fuel] + if ("oil_tanker" in flow_fuel) & ("crudeoil" not in flow_fuel): + basedf["value"] *= shipping_fuel_dict["oil_tanker"][flow_fuel] + + histdf = pd.concat([histdf, basedf]) + + histdf.to_csv( + os.path.join( + out_path, + tec, + "edit_files", + "flow_technology", + "historical_activity.csv", + ), + index=False, + ) + histdf.to_csv( + os.path.join( + out_path, + tec, + "bare_files", + "flow_technology", + "historical_activity.csv", + ), + index=False, + ) diff --git a/message_ix_models/tools/bilateralize/prepare_edit.py b/message_ix_models/tools/bilateralize/prepare_edit.py new file mode 100644 index 0000000000..67282b738b --- /dev/null +++ b/message_ix_models/tools/bilateralize/prepare_edit.py @@ -0,0 +1,1404 @@ +# -*- coding: utf-8 -*- +""" +Prepare edit files for bilateralize tool + +This script is the first step in implementing the bilateralize tool. +It generates empty (or default valued) parameters that are required for + bilateralization, specified by commodity. +This step is optional in a workflow; users can move directly to the next + step (2_bare_to_scenario). +""" + +# Import packages +import itertools +import logging +import os +import shutil +from pathlib import Path +from typing import Any, Dict + +import message_ix +import numpy as np +import pandas as pd +import yaml + +from message_ix_models.tools.bilateralize.calculate_distance import calculate_distance +from message_ix_models.tools.bilateralize.historical_calibration import ( + build_historical_price, +) +from message_ix_models.tools.bilateralize.mariteam_calibration import calibrate_mariteam +from message_ix_models.tools.bilateralize.pull_gem import import_gem +from message_ix_models.tools.bilateralize.utils import get_logger, load_config +from message_ix_models.util import package_data_path + + +# %% Generate folders if missing +def folders_for_trade(covered_tec: list[str]): + """ + Generate folders for each trade technology + """ + for tec in covered_tec: + tecpath = os.path.join(Path(package_data_path("bilateralize")), tec) + if not os.path.isdir(tecpath): + os.makedirs(tecpath) + pathlist = [ + os.path.join(tecpath, "edit_files"), + os.path.join(tecpath, "edit_files", "flow_technology"), + os.path.join(tecpath, "bare_files"), + os.path.join(tecpath, "bare_files", "flow_technology"), + ] + for f in pathlist: + if not os.path.isdir(f): + os.makedirs(f) + + +def define_networks( + log, + message_regions: str, + covered_tec: list[str], + config_dict: Dict[str, Dict[str, Any]], + data_path: Path, +): + """ + Define network dataframe + + Args: + message_regions: Regional resolution + """ + # Generate full combination of nodes to build technology-specific network + node_path = package_data_path("node", message_regions + ".yaml") + with open(node_path, "r", encoding="utf-8") as file_handler: + node_set = yaml.safe_load(file_handler) + node_set = [r for r in node_set.keys() if r not in ["World", "GLB"]] + + node_df = pd.DataFrame(itertools.product(node_set, node_set)) + node_df.columns = ["exporter", "importer"] + node_df = node_df[node_df["exporter"] != node_df["importer"]] + + network_setup = {} + + for tec in covered_tec: + log.info(f"Defining network for {tec}") + node_df_tec = node_df.copy() + + node_df_tec["export_technology"] = config_dict["trade_technology"][tec] + "_exp" + if config_dict["trade_tech_suffix"][tec] is not None: + node_df_tec["export_technology"] = ( + node_df_tec["export_technology"] + + "_" + + config_dict["trade_tech_suffix"][tec] + ) + + node_df_tec["export_technology"] = ( + node_df_tec["export_technology"] + + "_" + + node_df_tec["importer"].str.lower().str.split("_").str[1] + ) + + # If there are multiple trade "routes" (not flow, but decision) per technology + if (config_dict["trade_tech_number"][tec] is not None) & ( + config_dict["trade_tech_number"][tec] != 1 + ): + ndt_out = pd.DataFrame() + for i in list(range(1, config_dict["trade_tech_number"][tec] + 1)): + ndt = node_df_tec.copy() + ndt["export_technology"] = ndt["export_technology"] + "_" + str(i) + ndt_out = pd.concat([ndt_out, ndt]) + node_df_tec = ndt_out.copy() + + node_df_tec["import_technology"] = config_dict["trade_technology"][tec] + "_imp" + node_df_tec["INCLUDE? (No=0, Yes=1)"] = "" + + # Specify network (the function will stop to allow specification) + if config_dict["specify_network"][tec] is True: + try: + specify_network_tec = pd.read_csv( + os.path.join(data_path, tec, "specify_network_" + tec + ".csv") + ) + except FileNotFoundError: + node_df_tec.to_csv( + os.path.join(data_path, tec, "specify_network_" + tec + ".csv"), + index=False, + ) + raise Exception( + "The function stopped. Sheet specify_network_" + + tec + + ".csv has been generated. " + + "Fill in the specific pairs first and run again." + ) + if not any(specify_network_tec["INCLUDE? (No=0, Yes=1)"].notnull()): + raise Exception( + "The function stopped. Ensure that all values " + + "under 'INCLUDE? (No=0, Yes=1)' are filled" + ) + elif config_dict["specify_network"][tec] is False: + specify_network_tec = node_df_tec.copy() + specify_network_tec["INCLUDE? (No=0, Yes=1)"] = 1 + else: + raise Exception("Please use True or False.") + + network_setup[tec] = specify_network_tec[ + specify_network_tec["INCLUDE? (No=0, Yes=1)"] == 1 + ] + + return network_setup + # %% Build base parameter dataframe based on network dataframe + + +def build_parameterdf( + par_name: str, + network_df: pd.DataFrame, + col_values: dict, + common_years: dict | None = None, + common_cols: dict | None = None, + export_only: bool = False, +): + """ + Build parameter dataframes based on the specified network dataframe. + + Args: + par_name: Parameter name (e.g., capacity_factor) + network_df: Specified network dataframe + col_values: Values for other columns to populate as default + export_only: If True, only produces dataframe for export technology + """ + + if common_years is None: + common_years = dict( + year_vtg="broadcast", year_rel="broadcast", year_act="broadcast" + ) + if common_cols is None: + common_cols = dict(mode="M1", time="year", time_origin="year", time_dest="year") + + df_export = message_ix.make_df( + par_name, + node_loc=network_df["exporter"], + technology=network_df["export_technology"], + **col_values, + **common_years, + **common_cols, + ) + df = df_export.copy() + + if not export_only: + df_import = message_ix.make_df( + par_name, + node_loc=network_df["importer"], + technology=network_df["import_technology"], + **col_values, + **common_years, + **common_cols, + ) + df = pd.concat([df, df_import]) + + return df + + +# %% Generate input parameter (trade technology) +def build_input( + tec: str, + network_setup: dict, + config_dict: dict, + common_years: dict, + common_cols: dict, + parameter_outputs: dict, +): + """ + Generate input parameter (trade technology) + """ + # Trade Level (supply to piped/shipped) + df_input_trade = message_ix.make_df( + "input", + node_origin=network_setup[tec]["exporter"], + node_loc=network_setup[tec]["exporter"], + technology=network_setup[tec]["export_technology"], + commodity=config_dict["trade_commodity"][tec], + level=config_dict["export_level"][tec], + value=1, + unit=config_dict["trade_units"][tec], + **common_years, + **common_cols, + ) + + # Import Level (piped/shipped to import) + df_input_import = message_ix.make_df( + "input", + node_origin=network_setup[tec]["importer"], + node_loc=network_setup[tec]["importer"], + technology=network_setup[tec]["import_technology"], + commodity=config_dict["trade_commodity"][tec], + level=config_dict["trade_level"][tec], + value=1, + unit=config_dict["trade_units"][tec], + **common_years, + **common_cols, + ) + + df_input = pd.concat([df_input_trade, df_input_import]).drop_duplicates() + + parameter_outputs[tec]["trade"]["input"] = df_input + + return parameter_outputs + + +# %% Generate output parameter (trade technology) +def build_output( + tec: str, + network_setup: dict, + config_dict: dict, + common_years: dict, + common_cols: dict, + parameter_outputs: dict, +): + """ + Generate output parameter (trade technology) + """ + # Trade Level + df_output_trade = message_ix.make_df( + "output", + node_loc=network_setup[tec]["exporter"], + node_dest=network_setup[tec]["importer"], + technology=network_setup[tec]["export_technology"], + commodity=config_dict["trade_commodity"][tec], + level=config_dict["trade_level"][tec], + value=1, + unit=config_dict["trade_units"][tec], + **common_years, + **common_cols, + ) + + # Import Level + df_output_import = message_ix.make_df( + "output", + node_loc=network_setup[tec]["importer"], + node_dest=network_setup[tec]["importer"], + technology=network_setup[tec]["import_technology"], + commodity=config_dict["trade_commodity"][tec], + level=config_dict["import_level"][tec], + value=1, + unit=config_dict["trade_units"][tec], + **common_years, + **common_cols, + ) + + df_output = pd.concat([df_output_trade, df_output_import]).drop_duplicates() + + parameter_outputs[tec]["trade"]["output"] = df_output + + return parameter_outputs + + +# %% Generate technical lifetime parameter (trade technology) +def build_technical_lifetime( + tec: str, network_setup: dict, parameter_outputs: dict, **kwargs +): + """ + Generate technical lifetime parameter (trade technology) + """ + df_teclt = build_parameterdf( + "technical_lifetime", + network_df=network_setup[tec], + col_values=dict( + value=1, # Make 1 year by default + unit="y", + ), + ) + + parameter_outputs[tec]["trade"]["technical_lifetime"] = df_teclt.drop_duplicates() + + return parameter_outputs + + +# %% Generate historical activity parameter (trade technology) +def build_historical_activity( + tec: str, network_setup: dict, config_dict: dict, parameter_outputs: dict, **kwargs +): + """ + Generate costs for trade technology + """ + df_hist = pd.DataFrame() + + for y in list(range(2000, 2025, 5)): + ydf = build_parameterdf( + "historical_activity", + network_df=network_setup[tec], + col_values=dict(unit=config_dict["trade_units"][tec]), + ) + ydf["year_act"] = y + df_hist = pd.concat([df_hist, ydf]) + + parameter_outputs[tec]["trade"]["historical_activity"] = df_hist.drop_duplicates() + + return parameter_outputs + + +# %% Generate costs for trade technology +def build_costs( + tec: str, network_setup: dict, config_dict: dict, parameter_outputs: dict, **kwargs +): + """ + Generate costs for trade technology + """ + # Create base files: inv_cost, fix_cost, var_cost + for cost_par in ["inv_cost", "fix_cost", "var_cost"]: + df_cost = build_parameterdf( + cost_par, + network_df=network_setup[tec], + col_values=dict(unit="USD/" + config_dict["trade_units"][tec]), + ) + parameter_outputs[tec]["trade"][cost_par] = df_cost.drop_duplicates() + + return parameter_outputs + + +# %% Generate capacity factor parameter (trade technology) +def build_capacity_factor( + tec: str, network_setup: dict, config_dict: dict, parameter_outputs: dict, **kwargs +): + """ + Generate capacity factor parameter (trade technology) + """ + df_cf = build_parameterdf( + "capacity_factor", + network_df=network_setup[tec], + col_values=dict(value=1, unit="%"), + ) + + parameter_outputs[tec]["trade"]["capacity_factor"] = df_cf.drop_duplicates() + + return parameter_outputs + + +# %% Generate constraints for trade technology +def build_constraints( + tec: str, network_setup: dict, config_dict: dict, parameter_outputs: dict, **kwargs +): + """ + Generate constraints for trade technology + """ + for par_name in [ + "initial_activity", + "abs_cost_activity_soft", + "growth_activity", + "level_cost_activity_soft", + "soft_activity", + ]: + for t in ["lo", "up"]: + df_con = build_parameterdf( + par_name + "_" + t, + network_df=network_setup[tec], + col_values=dict(unit=config_dict["trade_units"][tec]), + ) + + if par_name == "growth_activity": + if t == "lo": + df_con["value"] = -0.05 + if t == "up": + df_con["value"] = 0.05 + + if par_name == "initial_activity": + if t == "lo": + df_con["value"] = 2 + if t == "up": + df_con["value"] = 2 + + df_con = df_con.drop_duplicates() + parameter_outputs[tec]["trade"][par_name + "_" + t] = df_con + + return parameter_outputs + + +def build_domestic_coalgas_relation( + tec: str, + network_setup: dict, + config_dict: dict, + common_years: dict, + common_cols: dict, + parameter_outputs: dict, + **kwargs, +): + """ + Generate domestic coal and gas relation parameter (trade technology) + """ + if tec in ["gas_piped"]: + for rel_act in ["domestic_coal", "domestic_gas"]: + df_rel = message_ix.make_df( + "relation_activity", + node_loc=network_setup[tec]["importer"], + node_rel=network_setup[tec]["importer"], + technology=network_setup[tec]["import_technology"], + commodity=config_dict["trade_commodity"][tec], + value=-1, + unit="???", + relation=rel_act, + **common_years, + **common_cols, + ) + df_rel = df_rel.drop_duplicates() + + parameter_outputs[tec]["trade"]["relation_activity_" + rel_act] = df_rel + + return parameter_outputs + + +# %% Generate emission factor parameter (trade technology) +def build_emission_factor( + tec: str, network_setup: dict, config_dict: dict, parameter_outputs: dict, **kwargs +): + """ + Generate emission factor parameter (trade technology) + """ + if config_dict["tracked_emissions"][tec] is not None: + df_ef = pd.DataFrame() + for emission_type in config_dict["tracked_emissions"][tec]: + df_ef_t = build_parameterdf( + "emission_factor", + network_df=network_setup[tec], + col_values=dict(unit=None, emission=emission_type), + ) + df_ef = pd.concat([df_ef, df_ef_t]) + + parameter_outputs[tec]["trade"]["emission_factor"] = df_ef.drop_duplicates() + + return parameter_outputs + + +# %% Generate CO2 relation accounting +def build_accounting_relations( + tec: str, + network_setup: dict, + config_dict: dict, + common_years: dict, + common_cols: dict, + parameter_outputs: dict, + **kwargs, +): + """ + Relations for accounting purposes: + CO2 emissions, primary energy total, aggregate exports, + regional exports, regional imports + + """ + # CO2 emissions + df_rel = message_ix.make_df( + "relation_activity", + node_loc=network_setup[tec]["exporter"], + node_rel=network_setup[tec]["exporter"], + technology=network_setup[tec]["export_technology"], + commodity=config_dict["trade_commodity"][tec], + unit="???", + relation="CO2_Emission", + **common_years, + **common_cols, + ) + df_rel = df_rel.drop_duplicates() + + parameter_outputs[tec]["trade"]["relation_activity_CO2_Emission"] = df_rel + + # Primary energy total + df_rel = message_ix.make_df( + "relation_activity", + node_loc=network_setup[tec]["exporter"], + node_rel=network_setup[tec]["exporter"], + technology=network_setup[tec]["export_technology"], + commodity=config_dict["trade_commodity"][tec], + value=-1, + unit="???", + relation="PE_total_traditional", + **common_years, + **common_cols, + ) + df_rel = df_rel.drop_duplicates() + + parameter_outputs[tec]["trade"]["relation_activity_PE_total_traditional"] = df_rel + + # Aggregate exports + df_rel = message_ix.make_df( + "relation_activity", + node_loc=network_setup[tec]["exporter"], + node_rel=network_setup[tec]["exporter"], + technology=network_setup[tec]["export_technology"], + commodity=config_dict["trade_commodity"][tec], + value=1, + unit="???", + relation=config_dict["trade_technology"][tec] + "_exp_global", + **common_years, + **common_cols, + ) + df_rel = df_rel.drop_duplicates() + + parameter_outputs[tec]["trade"]["relation_activity_global_aggregate"] = df_rel + + # Regional exports + rel_use = config_dict["trade_technology"][tec] + "_exp" + df_rel = message_ix.make_df( + "relation_activity", + node_loc=network_setup[tec]["exporter"], + node_rel=network_setup[tec]["exporter"], + technology=network_setup[tec]["export_technology"], + commodity=config_dict["trade_commodity"][tec], + value=1, + unit="???", + relation=rel_use, + **common_years, + **common_cols, + ) + df_rel = df_rel.drop_duplicates() + + parameter_outputs[tec]["trade"]["relation_activity_regional_exp"] = df_rel + + # Regional imports + rel_use = config_dict["trade_technology"][tec] + "_imp" + df_rel = message_ix.make_df( + "relation_activity", + node_loc=network_setup[tec]["importer"], + node_rel=network_setup[tec]["importer"], + technology=network_setup[tec]["import_technology"], + commodity=config_dict["trade_commodity"][tec], + value=1, + unit="???", + relation=rel_use, + **common_years, + **common_cols, + ) + df_rel = df_rel.drop_duplicates() + + parameter_outputs[tec]["trade"]["relation_activity_regional_imp"] = df_rel + + return parameter_outputs + + +# %% Generate flow technology input parameter +def build_flow_input( + flow_tec: str, + tec: str, + network_setup: dict, + config_dict: dict, + common_years: dict, + common_cols: dict, + parameter_outputs: dict, + message_regions: str = "R12", + **kwargs, +): + """ + Generate flow technology input parameter + """ + df_input = pd.DataFrame() + # List of commodity/material inputs + flow_inputs = config_dict["flow_fuel_input"][tec][flow_tec] + + # Build by commodity input + for c in flow_inputs: + if c in config_dict["flow_fuel_input"][tec][flow_tec]: + use_unit = "GWa" + elif c in config_dict["flow_material_input"][tec][flow_tec]: + use_unit = "Mt" + + # If pipelines, technologies and outputs are bilateral + if config_dict["flow_constraint"][tec] == "bilateral": + tec_use = ( + flow_tec + + "_" + + network_setup[tec]["importer"].str.lower().str.split("_").str[1] + ) + df_input_base = message_ix.make_df( + "input", + node_loc=network_setup[tec]["exporter"], + node_origin=network_setup[tec]["exporter"], + level=config_dict["export_level"][tec], + technology=tec_use, + commodity=c, + unit=use_unit, + **common_years, + **common_cols, + ) + df_input = pd.concat([df_input, df_input_base]) + + # If shipping, technologies and outputs are global with diff modes + elif config_dict["flow_constraint"][tec] == "global": + exp0 = network_setup[tec]["exporter"].str.replace(message_regions + "_", "") + imp0 = network_setup[tec]["importer"].str.replace(message_regions + "_", "") + mode_use = exp0 + "-" + imp0 + df_input_base = message_ix.make_df( + "input", + node_loc=network_setup[tec]["exporter"], + node_origin=network_setup[tec]["exporter"], + mode=mode_use, + level=config_dict["export_level"][tec], + technology=flow_tec, + commodity=c, + unit=use_unit, + time="year", + time_origin="year", + **common_years, + ) + if "shipped" in tec: + df_input_base["value"] = 1e-6 # Default is 1e-6 GWa + df_input = pd.concat([df_input, df_input_base]) + + # For shipped trade, set up bunker fuels + if config_dict["bunker_technology"][tec] is not None: + # Global bunker + df_input_gbunk = df_input.copy() + df_input_gbunk["technology"] = "bunker_global" + "_" + c + df_input_gbunk["mode"] = "M1" + df_input_gbunk["value"] = 1 # Default is 1 GWa + df_input = pd.concat([df_input, df_input_gbunk]) + + df_input = df_input.drop_duplicates() + input_out = pd.concat([parameter_outputs[tec]["flow"]["input"], df_input]) + parameter_outputs[tec]["flow"]["input"] = input_out + + return parameter_outputs + + +# %% Generate flow technology output parameter +def build_flow_output( + flow_tec: str, + tec: str, + network_setup: dict, + config_dict: dict, + common_years: dict, + common_cols: dict, + parameter_outputs: dict, + message_regions: str = "R12", + **kwargs, +): + """ + Generate flow technology output parameter + """ + df_output = pd.DataFrame() + # If pipelines technologies and outputs are bilateral + if config_dict["flow_constraint"][tec] == "bilateral": + df_output_base = message_ix.make_df( + "output", + node_loc=network_setup[tec]["exporter"], + node_dest=network_setup[tec]["exporter"], + technology=flow_tec + + "_" + + network_setup[tec]["importer"].str.lower().str.split("_").str[1], + commodity=config_dict["flow_commodity_output"][tec] + + "_" + + network_setup[tec]["importer"].str.lower().str.split("_").str[1], + unit=config_dict["flow_units"][tec], + level=config_dict["trade_level"][tec], + **common_years, + **common_cols, + ) + df_output = pd.concat([df_output, df_output_base]) + + # If shipping, technologies and outputs are global with diff modes + elif config_dict["flow_constraint"][tec] == "global": + df_output_base = message_ix.make_df( + "output", + node_loc=network_setup[tec]["exporter"], + node_dest=message_regions + "_GLB", + mode="M1", + technology=flow_tec, + commodity=config_dict["flow_commodity_output"][tec], + unit=config_dict["flow_units"][tec], + level=config_dict["trade_level"][tec], + time="year", + time_dest="year", + **common_years, + ) + df_output = pd.concat([df_output, df_output_base]) + + # For shipped trade, set up bunker fuels + if config_dict["bunker_technology"][tec] is not None: + # Global bunker + df_output_gbunk = df_output.copy() + df_output_gbunk["technology"] = ( + "bunker_global_" + config_dict["flow_fuel_input"][tec][flow_tec][0] + ) + df_output_gbunk["node_dest"] = message_regions + "_GLB" + df_output_gbunk["level"] = "bunker" + df_output_gbunk["unit"] = "GWa" + df_output_gbunk["mode"] = "M1" + df_output_gbunk["commodity"] = config_dict["flow_fuel_input"][tec][flow_tec][0] + df_output = pd.concat([df_output, df_output_gbunk]) + + df_output = df_output.drop_duplicates() + + df_output["value"] = 1 + output_out = pd.concat([parameter_outputs[tec]["flow"]["output"], df_output]) + parameter_outputs[tec]["flow"]["output"] = output_out + + return parameter_outputs + + +# %% Generate flow technology capacity constraints parameter +def build_flow_capacity_constraints( + par: str, + flow_tec: str, + tec: str, + network_setup: dict, + config_dict: dict, + common_years: dict, + common_cols: dict, + parameter_outputs: dict, + **kwargs, +): + """ + Generate flow technology capacity constraints parameter + """ + for t in ["lo", "up"]: + if config_dict["flow_constraint"][tec] == "bilateral": + df_con = message_ix.make_df( + par + "_" + t, + node_loc=network_setup[tec]["exporter"], + technology=flow_tec + + "_" + + network_setup[tec]["importer"].str.lower().str.split("_").str[1], + unit=config_dict["flow_units"][tec], + **common_years, + **common_cols, + ) + elif config_dict["flow_constraint"][tec] == "global": + df_con = message_ix.make_df( + par + "_" + t, + node_loc=network_setup[tec]["exporter"], + technology=flow_tec, + unit=config_dict["flow_units"][tec], + **common_years, + **common_cols, + ) + + con_out = pd.concat([parameter_outputs[tec]["flow"][par + "_" + t], df_con]) + parameter_outputs[tec]["flow"][par + "_" + t] = con_out + + return parameter_outputs + + +# %% Generate flow technology costs parameter +def build_flow_FIcosts( + flow_tec: str, + tec: str, + network_setup: dict, + config_dict: dict, + common_years: dict, + common_cols: dict, + parameter_outputs: dict, + **kwargs, +): + """ + Generate flow technology costs parameter + """ + for cost_par in ["fix_cost", "inv_cost"]: + if config_dict["flow_constraint"][tec] == "bilateral": + tec_use = ( + flow_tec + + "_" + + network_setup[tec]["importer"].str.lower().str.split("_").str[1] + ) + df_cost = message_ix.make_df( + cost_par, + node_loc=network_setup[tec]["exporter"], + technology=tec_use, + unit="USD/" + config_dict["flow_units"][tec], + **common_years, + **common_cols, + ) + elif config_dict["flow_constraint"][tec] == "global": + df_cost = message_ix.make_df( + cost_par, + node_loc=network_setup[tec]["exporter"], + technology=flow_tec, + unit="USD/" + config_dict["flow_units"][tec], + **common_years, + **common_cols, + ) + df_cost = df_cost.drop_duplicates() + + if "shipped" in tec: + df_cost["value"] = 0.04 # Default is 0.04 USD/GWa + + cost_out = pd.concat([parameter_outputs[tec]["flow"][cost_par], df_cost]) + parameter_outputs[tec]["flow"][cost_par] = cost_out + + return parameter_outputs + + +def build_flow_Vcosts( + flow_tec: str, + tec: str, + network_setup: dict, + config_dict: dict, + common_years: dict, + common_cols: dict, + parameter_outputs: dict, + message_regions: str = "R12", + **kwargs, +): + """ + Generate flow technology variable costs parameter + """ + if config_dict["flow_constraint"][tec] == "bilateral": + tec_use = ( + flow_tec + + "_" + + network_setup[tec]["importer"].str.lower().str.split("_").str[1] + ) + df_vcost_base = message_ix.make_df( + "var_cost", + node_loc=network_setup[tec]["exporter"], + technology=tec_use, + unit="USD/" + config_dict["flow_units"][tec], + **common_years, + **common_cols, + ) + + elif config_dict["flow_constraint"][tec] == "global": + exp0 = network_setup[tec]["exporter"].str.replace(message_regions + "_", "") + imp0 = network_setup[tec]["importer"].str.replace(message_regions + "_", "") + mode_use = exp0 + "-" + imp0 + + df_vcost_base = message_ix.make_df( + "var_cost", + node_loc=network_setup[tec]["exporter"], + mode=mode_use, + technology=flow_tec, + unit="USD/" + config_dict["flow_units"][tec], + time="year", + **common_years, + ) + if "shipped" in tec: + df_vcost_base["value"] = 0.002 # Default is 0.002 USD/Mt-km + + vcost_out = pd.concat([parameter_outputs[tec]["flow"]["var_cost"], df_vcost_base]) + parameter_outputs[tec]["flow"]["var_cost"] = vcost_out + + return parameter_outputs + + +# %% Generate flow technology capacity factor parameter +def build_flow_capacity_factor( + flow_tec: str, + tec: str, + network_setup: dict, + config_dict: dict, + common_years: dict, + common_cols: dict, + parameter_outputs: dict, + **kwargs, +): + """ + Generate flow technology capacity factor parameter + """ + df_cf_base = build_parameterdf( + "capacity_factor", + network_df=network_setup[tec], + col_values=dict(value=1, unit="%"), + export_only=True, + ) + + if config_dict["flow_constraint"][tec] == "bilateral": + df_cf_base["technology"] = ( + flow_tec + "_" + df_cf_base["technology"].str.lower().str.split("_").str[-1] + ) + elif config_dict["flow_constraint"][tec] == "global": + df_cf_base["technology"] = flow_tec + + df_cf_base = df_cf_base.drop_duplicates() + + if config_dict["bunker_technology"][tec] is not None: + # Add global bunker fuel technology + bdf = df_cf_base.copy() + bdf["technology"] = ( + "bunker_global_" + config_dict["flow_fuel_input"][tec][flow_tec][0] + ) + df_cf_base = pd.concat([df_cf_base, bdf]) + + cf_out = pd.concat([parameter_outputs[tec]["flow"]["capacity_factor"], df_cf_base]) + parameter_outputs[tec]["flow"]["capacity_factor"] = cf_out + + return parameter_outputs + + +# %% Generate flow technology technical lifetime parameter +def build_flow_technical_lifetime( + flow_tec: str, + tec: str, + network_setup: dict, + config_dict: dict, + common_years: dict, + common_cols: dict, + parameter_outputs: dict, + **kwargs, +): + """ + Generate flow technology technical lifetime parameter + """ + df_teclt_base = build_parameterdf( + "technical_lifetime", + network_df=network_setup[tec], + col_values=dict( + value=20, # Default is 20y + unit="y", + ), + export_only=True, + ) + + if config_dict["flow_constraint"][tec] == "bilateral": + df_teclt_base["technology"] = ( + flow_tec + + "_" + + df_teclt_base["technology"].str.lower().str.split("_").str[-1] + ) + elif config_dict["flow_constraint"][tec] == "global": + df_teclt_base["technology"] = flow_tec + + df_teclt_base = df_teclt_base.drop_duplicates() + + if config_dict["bunker_technology"][tec] is not None: + # Add global bunker fuel technology + bdf = df_teclt_base.copy() + bdf["technology"] = ( + "bunker_global_" + config_dict["flow_fuel_input"][tec][flow_tec][0] + ) + df_teclt_base = pd.concat([df_teclt_base, bdf]) + + teclt_out = pd.concat( + [parameter_outputs[tec]["flow"]["technical_lifetime"], df_teclt_base] + ) + parameter_outputs[tec]["flow"]["technical_lifetime"] = teclt_out + + return parameter_outputs + + +# %% Flow as trade input +def flow_as_trade_input( + flow_tec: str, + tec: str, + network_setup: dict, + config_dict: dict, + common_years: dict, + common_cols: dict, + parameter_outputs: dict, + message_regions: str, + data_path: Path, +): + """ + Generate flow technology as trade technology input parameter + """ + if config_dict["flow_constraint"][tec] == "bilateral": + df_input_flow = message_ix.make_df( + "input", + node_loc=network_setup[tec]["exporter"], + node_origin=network_setup[tec]["exporter"], + technology=network_setup[tec]["export_technology"], + commodity=config_dict["flow_commodity_output"][tec] + + "_" + + network_setup[tec]["importer"].str.lower().str.split("_").str[1], + unit=config_dict["flow_units"][tec], + level=config_dict["trade_level"][tec], + **common_years, + **common_cols, + ).drop_duplicates() + elif config_dict["flow_constraint"][tec] == "global": + df_input_flow = message_ix.make_df( + "input", + node_loc=network_setup[tec]["exporter"], + node_origin=message_regions + "_GLB", + technology=network_setup[tec]["export_technology"], + commodity=config_dict["flow_commodity_output"][tec], + unit=config_dict["flow_units"][tec], + level=config_dict["trade_level"][tec], + **common_years, + **common_cols, + ).drop_duplicates() + + # For shipped commodities calculate capacities based on distance & energy content + distance_df = pd.read_csv( + os.path.join(data_path, "distances", message_regions + "_distances.csv") + ) + energycontent_df = pd.read_excel(os.path.join(data_path, "specific_energy.xlsx")) + if config_dict["trade_units"][tec] == "GWa": + tradecom = config_dict["trade_commodity"][tec] + speccont = energycontent_df[energycontent_df["Commodity"] == tradecom] + speccont = speccont["Specific Energy (GWa/Mt)"].reset_index(drop=True)[0] + else: # e.g., for materials + speccont = 1 + + mult_df = distance_df.copy() + mult_df["node_loc"] = mult_df["Node1"] + mult_df["technology"] = ( + config_dict["trade_technology"][tec] + + "_exp_" + + mult_df["Node2"].str.lower().str.split("_").str[1] + ) + mult_df["energy_content"] = speccont + mult_df["multiplier"] = mult_df["Distance_km"] / mult_df["energy_content"] + mult_df = mult_df[["node_loc", "technology", "multiplier"]].drop_duplicates() + + df_input_flow = df_input_flow.merge( + mult_df, + left_on=["node_loc", "technology"], + right_on=["node_loc", "technology"], + how="left", + ) + df_input_flow["value"] = df_input_flow["multiplier"] + df_input_flow = df_input_flow[message_ix.make_df("input").columns] + + input_out = pd.concat([parameter_outputs[tec]["trade"]["input"], df_input_flow]) + parameter_outputs[tec]["trade"]["input"] = input_out + + return parameter_outputs + + +# %% Export edit files +def export_edit_files( + covered_tec: list[str], + log: logging.Logger, + data_path: Path, + parameter_outputs: dict, +): + """ + Export edit files + """ + for tec in covered_tec: + log.info(f"Exporting trade parameters for {tec}") + for parname in parameter_outputs[tec]["trade"].keys(): + parameter_outputs[tec]["trade"][parname].to_csv( + os.path.join(data_path, tec, "edit_files", parname + ".csv"), + index=False, + ) + log.info(f"...trade {parname}") + for parname in parameter_outputs[tec]["flow"].keys(): + parameter_outputs[tec]["flow"][parname].to_csv( + os.path.join( + data_path, tec, "edit_files", "flow_technology", parname + ".csv" + ), + index=False, + ) + log.info(f"...flow {parname}") + + ## Transfer files from edit to bare if they do not already exist + for tec in covered_tec: + reqpar_list = [ + os.path.join("capacity_factor.csv"), + os.path.join("input.csv"), + os.path.join("output.csv"), + os.path.join("technical_lifetime.csv"), + os.path.join( + "relation_activity_regional_exp.csv" + ), # necessary for legacy reporting + os.path.join("flow_technology", "capacity_factor.csv"), + os.path.join("flow_technology", "input.csv"), + os.path.join("flow_technology", "output.csv"), + os.path.join("flow_technology", "technical_lifetime.csv"), + ] + for reqpar in reqpar_list: + if not os.path.isfile(os.path.join(data_path, tec, "bare_files", reqpar)): + base_file = os.path.join(data_path, tec, "edit_files", reqpar) + dest_file = os.path.join(data_path, tec, "bare_files", reqpar) + shutil.copy2(base_file, dest_file) + log.info(f"Copied file from edit to bare: {reqpar}") + ## Transfer cost parameters for flow technologies using shipping from edit to bare + for tec in covered_tec: + if "shipped" in tec: + required_parameters = [ + os.path.join("flow_technology", "var_cost.csv"), + os.path.join("flow_technology", "inv_cost.csv"), + ] + for reqpar in required_parameters: + if not os.path.isfile( + os.path.join(data_path, tec, "bare_files", reqpar) + ): + base_file = os.path.join(data_path, tec, "edit_files", reqpar) + dest_file = os.path.join(data_path, tec, "bare_files", reqpar) + shutil.copy2(base_file, dest_file) + log.info(f"Copied file from edit to bare: {reqpar}") + + +# %% Main function to generate bare sheets +def generate_edit_files( + log, + project_name: str | None = None, + config_name: str | None = None, + message_regions: str = "R12", +): + """ + Generate bare sheets to collect required parameters + + Args: + log (log, required): Log file to track progress + project_name (str, optional): Project name (message_ix_models/project/[THIS]) + config_name (str, optional): Name of the config file. + If None, uses default config from data/bilateralize/config_default.yaml + message_regions (str, optional): Default is R12 regionality + """ + data_path = package_data_path("bilateralize") + data_path = Path(os.path.join(os.path.dirname(data_path), "bilateralize")) + + # Load config + log.info("Loading config file") + config_base, config_path, config_tec = load_config( + project_name, config_name, load_tec_config=True + ) + + # Retrieve config sections + message_regions = config_base.get("scenario", {}).get("regions") + covered_tec = config_base.get("covered_trade_technologies", {}) + + config_dict: Dict[str, Dict[str, Any]] = {} + for tec in covered_tec: + tec_dict = config_tec.get(tec).get(tec + "_trade", {}) + for k in tec_dict.keys(): + if k not in config_dict.keys(): + config_dict[k] = {} + config_dict[k][tec] = tec_dict[k] + + # Create folders for trade technologies (if missing) + folders_for_trade(covered_tec) + + # Generate full combination of nodes to build technology-specific network + network_setup = define_networks( + log, message_regions, covered_tec, config_dict, data_path + ) + + # Common values across parameters + common_years = dict( + year_vtg="broadcast", year_act="broadcast", year_rel="broadcast" + ) + common_cols = dict(mode="M1", time="year", time_origin="year", time_dest="year") + + parameter_outputs: Dict[str, Dict[str, Any]] = {} # This will get populated + for tec in covered_tec: + parameter_outputs[tec] = dict(trade=dict(), flow=dict()) + + # Populate parameter_outputs dictionaries + for tec in covered_tec: + tec_args = { + "tec": tec, + "network_setup": network_setup, + "config_dict": config_dict, + "common_years": common_years, + "common_cols": common_cols, + "parameter_outputs": parameter_outputs, + } + + # Trade input + parameter_outputs = build_input(**tec_args) + # Trade output + parameter_outputs = build_output(**tec_args) + + # Trade technical lifetime + parameter_outputs = build_technical_lifetime(**tec_args) + + # Trade historical activity + parameter_outputs = build_historical_activity(**tec_args) + + # Trade costs + parameter_outputs = build_costs(**tec_args) + + # Trade capacity factor + parameter_outputs = build_capacity_factor(**tec_args) + + # Trade constraints + parameter_outputs = build_constraints(**tec_args) + + # Trade emission factor + parameter_outputs = build_emission_factor(**tec_args) + + # For gas- imports require relation to domestic_coal and domestic_gas + parameter_outputs = build_domestic_coalgas_relation(**tec_args) + + # Accounting relations + parameter_outputs = build_accounting_relations(**tec_args) + + ## FLOW TECHNOLOGY ## + ##################### + # Create file: Input + for tec in covered_tec: + parameter_outputs[tec]["flow"]["input"] = pd.DataFrame() + parameter_outputs[tec]["flow"]["output"] = pd.DataFrame() + + for par in ["growth_new_capacity", "initial_new_capacity"]: + parameter_outputs[tec]["flow"][par + "_up"] = pd.DataFrame() + parameter_outputs[tec]["flow"][par + "_lo"] = pd.DataFrame() + + parameter_outputs[tec]["flow"]["fix_cost"] = pd.DataFrame() + parameter_outputs[tec]["flow"]["inv_cost"] = pd.DataFrame() + parameter_outputs[tec]["flow"]["var_cost"] = pd.DataFrame() + parameter_outputs[tec]["flow"]["capacity_factor"] = pd.DataFrame() + parameter_outputs[tec]["flow"]["technical_lifetime"] = pd.DataFrame() + + for flow_tec in config_dict["flow_technologies"][tec]: + flow_args = { + "flow_tec": flow_tec, + "tec": tec, + "network_setup": network_setup, + "config_dict": config_dict, + "common_years": common_years, + "common_cols": common_cols, + "parameter_outputs": parameter_outputs, + "message_regions": message_regions, + "data_path": data_path, + } + + # Flow input + parameter_outputs = build_flow_input(**flow_args) + + # Flow output + parameter_outputs = build_flow_output(**flow_args) + + # Flow capacity constraints + for par in ["growth_new_capacity", "initial_new_capacity"]: + parameter_outputs = build_flow_capacity_constraints( + par=par, **flow_args + ) + + # Flow costs + parameter_outputs = build_flow_FIcosts(**flow_args) + parameter_outputs = build_flow_Vcosts(**flow_args) + + # Flow capacity factor + parameter_outputs = build_flow_capacity_factor(**flow_args) + + # Flow technical lifetime + parameter_outputs = build_flow_technical_lifetime(**flow_args) + + # Add flow technology outputs as an input into the trade technology + parameter_outputs = flow_as_trade_input(**flow_args) + + ## Export files + export_edit_files(covered_tec, log, data_path, parameter_outputs) + + +def prepare_edit_files( + project_name: str | None = None, + config_name: str | None = None, + P_access: bool = True, +): + """ + Prepare edit files for bilateralize tool + + Args: + project_name: Name of the project (e.g., 'newpathways') + config_name: Name of the config file (e.g., 'config.yaml') + P_access: User has access to P: drive + """ + # Get logger + log = get_logger(__name__) + + # Bring in configuration + config, config_path = load_config( + project_name=project_name, config_name=config_name + ) + + covered_tec = config["covered_trade_technologies"] + message_regions = config["scenario"]["regions"] + + data_path = package_data_path("bilateralize") + data_path = Path(os.path.join(os.path.dirname(data_path), "bilateralize")) + + # Calculate distances + calculate_distance(message_regions) + + # Generate bare sheets + generate_edit_files( + log=log, + project_name=project_name, + config_name=config_name, + message_regions=message_regions, + ) + + # Import calibration files from Global Energy Monitor + if P_access is True: + import_gem( + input_file="GEM-GGIT-Gas-Pipelines-2024-12.csv", + # input_sheet="Gas Pipelines 2024-12-17", + trade_technology="gas_piped", + flow_technology="gas_pipe", + flow_commodity="gas_pipeline_capacity", + project_name=project_name, + config_name=config_name, + ) + + for tradetec in ["crudeoil_piped", "foil_piped", "loil_piped"]: + import_gem( + input_file="GEM-GOIT-Oil-NGL-Pipelines-2025-03.csv", + # input_sheet="Pipelines", + trade_technology=tradetec, + flow_technology="oil_pipe", + flow_commodity="oil_pipeline_capacity", + project_name=project_name, + config_name=config_name, + ) + + # Add MariTEAM calibration for maritime shipping + calibrate_mariteam( + covered_tec, + message_regions, + project_name=project_name, + config_name=config_name, + ) + + # Add variable costs for shipped commodities + costdf = build_historical_price( + message_regions, project_name=project_name, config_name=config_name + ) + costdf["technology"] = costdf["technology"].str.replace("ethanol_", "eth_") + costdf["technology"] = costdf["technology"].str.replace("fueloil_", "foil_") + + for tec in [i for i in covered_tec if i != "gas_piped"]: + log.info("Add fix cost for " + tec) + + if "piped" in tec: + tec_shipped = tec.replace("piped", "shipped") + add_df = costdf[costdf["technology"].str.contains(tec_shipped)].copy() + add_df["technology"] = add_df["technology"].str.replace( + "shipped", "piped" + ) + else: + add_df = costdf[costdf["technology"].str.contains(tec)] + + col_list = add_df.columns + mean_cost = add_df["value"].mean() + + input_df = pd.read_csv( + os.path.join(data_path, tec, "edit_files", "input.csv") + ) + input_df = input_df[ + ["node_loc", "technology", "year_act", "year_vtg"] # "mode", "time" + ].drop_duplicates() + add_df = input_df.merge( + add_df, + left_on=[ + "node_loc", + "technology", + "year_act", + "year_vtg", + ], + right_on=[ + "node_loc", + "technology", + "year_act", + "year_vtg", + ], + how="left", + ) + + add_df["value"] = np.where( + add_df["value"].isnull(), mean_cost, add_df["value"] + ) + add_df["value"] = round(add_df["value"] / 5, 0) + + add_df["unit"] = "USD/GWa" + + add_df = add_df[col_list] + + add_df = add_df[~add_df["technology"].str.contains("_imp")] + + add_df = add_df[ + ["node_loc", "technology", "year_act", "year_vtg", "value", "unit"] + ] + add_df = add_df.drop_duplicates() + + add_df.to_csv( + os.path.join(data_path, tec, "edit_files", "fix_cost.csv"), index=False + ) + add_df.to_csv( + os.path.join(data_path, tec, "bare_files", "fix_cost.csv"), index=False + ) diff --git a/message_ix_models/tools/bilateralize/pull_gem.py b/message_ix_models/tools/bilateralize/pull_gem.py new file mode 100644 index 0000000000..93b5f5c719 --- /dev/null +++ b/message_ix_models/tools/bilateralize/pull_gem.py @@ -0,0 +1,385 @@ +# -*- coding: utf-8 -*- +""" +Historical Calibration +""" + +# Import packages +import os +from pathlib import Path + +import message_ix +import numpy as np +import pandas as pd +import yaml + +from message_ix_models.tools.bilateralize.utils import load_config +from message_ix_models.util import package_data_path + +oil_pipeline_file = "GEM-GOIT-Oil-NGL-Pipelines-2025-03.xlsx" +oil_pipeline_sheet = "Pipelines" + +gas_pipeline_file = "GEM-GGIT-Gas-Pipelines-2024-12.xlsx" +gas_pipeline_sheet = "Gas Pipelines 2024-12-17" + + +# Set up MESSAGE regions +def gem_region(project_name: str | None = None, config_name: str | None = None): + """ + Set up MESSAGE regions + + Args: + project_name: Name of project + config_name: Name of config file + """ + + config, config_path = load_config( + project_name=project_name, config_name=config_name + ) + with open(config_path, "r", encoding="utf-8") as f: + config = yaml.safe_load(f) + message_regions = config["scenario"]["regions"] + + full_path = package_data_path("node", message_regions + ".yaml") + with open(full_path, "r", encoding="utf-8") as f: + message_regions = yaml.safe_load(f) + message_regions_list = [ + r for r in message_regions.keys() if r not in ["World", "GLB"] + ] + + return message_regions_list, message_regions + + +# Import files +def import_gem( + input_file: str | Path, + # input_sheet: str, + trade_technology: str, + flow_technology: str, + flow_commodity: str, + project_name: str | None = None, + config_name: str | None = None, + first_model_year: int = 2030, +): + """ + Import Global Energy Monitor data + + Args: + input_file: Name of input file + input_sheet: Name of input sheet + trade_technology: Name of trade technology + flow_technology: Name of flow technology + flow_commodity: Name of flow commodity + project_name: Name of project + config_name: Name of config file + first_model_year: First model year + """ + + # Pull in configuration + config, config_path = load_config( + project_name=project_name, config_name=config_name + ) + p_drive = config["p_drive_location"] + + # Data paths + data_path = os.path.join(p_drive, "MESSAGE_trade") + gem_path = os.path.join(data_path, "Global Energy Monitor") + + # df = pd.read_excel(os.path.join(gem_path, input_file), sheet_name=input_sheet) + df = pd.read_csv(os.path.join(gem_path, input_file), encoding="utf-8") + + df = df[df["StopYear"].isnull()] # Only continuing projects + + df = df[ + [ + "StartYear1", + "StartCountry", + "EndCountry", + "CapacityBOEd", + "CostUSD", + "LengthMergedKm", + ] + ].drop_duplicates() + + # Clean up country codes + cw = pd.read_csv(os.path.join(gem_path, "country_crosswalk.csv")) + for i in ["Start", "End"]: + df = df.merge(cw, left_on=i + "Country", right_on="GEM Country", how="left") + df = df.rename(columns={"ISO": i + "ISO"}) + + # Add MESSAGE regions + message_regions_list, message_regions = gem_region(project_name, config_name) + df["EXPORTER"] = "" + df["IMPORTER"] = "" + for r in message_regions_list: + df["EXPORTER"] = np.where( + df["StartISO"].isin(message_regions[r]["child"]), r, df["EXPORTER"] + ) + df["IMPORTER"] = np.where( + df["EndISO"].isin(message_regions[r]["child"]), r, df["IMPORTER"] + ) + + # Collapse + df["CapacityBOEd"] = df["CapacityBOEd"].replace("--", "0", regex=True).astype(float) + df["CostUSD"] = df["CostUSD"].replace("--", "0", regex=True).astype(float) + df["LengthMergedKm"] = ( + df["LengthMergedKm"].replace("--", "0", regex=True).astype(float) + ) + + df = df[(~df["CapacityBOEd"].isnull()) & (~df["CostUSD"].isnull())] + + df = df[df["StartYear1"] < first_model_year] # No planned capacity + df["YEAR"] = ( + round(df["StartYear1"].astype(float) / 5) * 5 + ) # Round year to the nearest 5 + + df = ( + df.groupby(["EXPORTER", "IMPORTER", "YEAR"])[ + ["CapacityBOEd", "CostUSD", "LengthMergedKm"] + ] + .sum() + .reset_index() + ) + + # Convert units + df["Capacity (BOEa)"] = df["CapacityBOEd"] * 365 + df["Capacity (TJ)"] = df["Capacity (BOEa)"] * 0.006 # BOEa to TJ + df["Capacity (GWa)"] = df["Capacity (TJ)"] * (3.1712 * 1e-5) # TJ to GWa + + # Generate investment costs + df["InvCost (USD/km)"] = (df["CostUSD"]) / df["LengthMergedKm"] + # TODO: Add industry-specific deflators + + # Generate capacity + df["Capacity (GWa/km)"] = (df["Capacity (GWa)"]) / df["LengthMergedKm"] + + # Cut down + df = df[df["EXPORTER"] != df["IMPORTER"]] + df = df[(df["EXPORTER"] != "") & (df["IMPORTER"] != "")] + + # Base file with all historical timesteps + hist_base = pd.DataFrame() + for y in list(range(2000, 2030, 5)): + ydf = df[["EXPORTER", "IMPORTER"]].drop_duplicates().copy() + ydf["YEAR"] = y + hist_base = pd.concat([hist_base, ydf]) + hist_base = hist_base[(hist_base["EXPORTER"] != "") & (hist_base["IMPORTER"] != "")] + + df = hist_base.merge( + df, + left_on=["EXPORTER", "IMPORTER", "YEAR"], + right_on=["EXPORTER", "IMPORTER", "YEAR"], + how="outer", + ) # Set 2005 to 0 if missing + for c in [c for c in df.columns if c not in ["EXPORTER", "IMPORTER", "YEAR"]]: + df[c] = np.where((df[c].isnull()) & (df["YEAR"] == 2000), 0, df[c]) + + df["trade_technology"] = trade_technology + df["flow_technology"] = flow_technology + + # Output to trade_technology + export_dir = package_data_path("bilateralize", trade_technology) + gem_dir_out = os.path.join(os.path.dirname(export_dir), trade_technology, "GEM") + + trade_dir = os.path.join(os.path.dirname(gem_dir_out), "edit_files") + flow_dir = os.path.join( + os.path.dirname(gem_dir_out), "edit_files", "flow_technology" + ) + trade_dir_out = os.path.join(os.path.dirname(gem_dir_out), "bare_files") + flow_dir_out = os.path.join( + os.path.dirname(gem_dir_out), "bare_files", "flow_technology" + ) + if not os.path.isdir(gem_dir_out): + os.makedirs(Path(gem_dir_out)) + + df.to_csv(os.path.join(gem_dir_out, "GEM.csv")) + + # Investment Costs + inv_cost = ( + df.groupby(["EXPORTER", "IMPORTER"])[["CostUSD", "LengthMergedKm"]] + .sum() + .reset_index() + ) + inv_cost["InvCost (USD/km)"] = (inv_cost["CostUSD"]) / inv_cost["LengthMergedKm"] + inv_cost = inv_cost[["EXPORTER", "IMPORTER", "InvCost (USD/km)"]].drop_duplicates() + inv_cost["node_loc"] = inv_cost["EXPORTER"] + inv_cost["technology"] = ( + flow_technology + "_" + inv_cost["IMPORTER"].str.lower().str.split("_").str[-1] + ) + inv_cost["value_update"] = inv_cost["InvCost (USD/km)"] / 1e6 # in MUSD/km + inv_cost = inv_cost[["node_loc", "technology", "value_update"]] + inv_cost.to_csv(os.path.join(gem_dir_out, "inv_cost_GEM.csv"), index=False) + + basedf = pd.read_csv(os.path.join(flow_dir, "inv_cost.csv")) + basedf["value"] = 10 + inv_cost = basedf.merge( + inv_cost, + left_on=["node_loc", "technology"], + right_on=["node_loc", "technology"], + how="left", + ) + inv_cost["value"] = np.where( + inv_cost["value_update"] > 0, + round(inv_cost["value_update"], 0), + inv_cost["value"], + ) + inv_cost["year_vtg"] = "broadcast" + inv_cost["unit"] = "USD/km" + inv_cost = inv_cost[ + ["node_loc", "technology", "year_vtg", "value", "unit"] + ].drop_duplicates() + inv_cost.to_csv(os.path.join(flow_dir, "inv_cost.csv"), index=False) + inv_cost.to_csv(os.path.join(flow_dir_out, "inv_cost.csv"), index=False) + + # Historical activity (flow) + hist_act = df[["EXPORTER", "IMPORTER", "YEAR", "LengthMergedKm"]].drop_duplicates() + hist_act["LengthMergedKm"] = np.where( + hist_act["LengthMergedKm"].isnull(), 0, hist_act["LengthMergedKm"] + ) + hist_act["LengthMergedKm"] = hist_act.groupby(["EXPORTER", "IMPORTER"])[ + "LengthMergedKm" + ].transform(pd.Series.cumsum) + hist_act["node_loc"] = hist_act["EXPORTER"] + hist_act["technology"] = ( + flow_technology + "_" + hist_act["IMPORTER"].str.lower().str.split("_").str[-1] + ) + hist_act["value"] = round(hist_act["LengthMergedKm"], 0) + hist_act["year_act"] = hist_act["YEAR"].astype(int) + hist_act["unit"] = "km" + hist_act["mode"] = "M1" + hist_act["time"] = "year" + hist_act = hist_act[ + ["node_loc", "technology", "year_act", "value", "unit", "mode", "time"] + ] + hist_act.to_csv( + os.path.join(gem_dir_out, "historical_activity_GEM.csv"), index=False + ) + hist_act.to_csv(os.path.join(flow_dir, "historical_activity.csv"), index=False) + hist_act.to_csv(os.path.join(flow_dir_out, "historical_activity.csv"), index=False) + + # Historical activity (trade for oil pipelines only) + # Share of oil pipeline capacity used for commodity (crude/light oil/fuel oil) + share_oil_pipelines = {"crudeoil_piped": 0.8, "loil_piped": 0.1, "foil_piped": 0.1} + + if trade_technology in ["crudeoil_piped", "loil_piped", "foil_piped"]: + hist_tra = df[ + ["EXPORTER", "IMPORTER", "YEAR", "Capacity (GWa)"] + ].drop_duplicates() + hist_tra["Capacity (GWa)"] = np.where( + hist_tra["Capacity (GWa)"].isnull(), 0, hist_tra["Capacity (GWa)"] + ) + hist_tra["Capacity (GWa)"] = hist_tra.groupby(["EXPORTER", "IMPORTER"])[ + "Capacity (GWa)" + ].transform(pd.Series.cumsum) + hist_tra["node_loc"] = hist_tra["EXPORTER"] + hist_tra["technology"] = ( + trade_technology + + "_exp_" + + hist_tra["IMPORTER"].str.lower().str.split("_").str[-1] + ) + + hist_tra["value"] = round(hist_tra["Capacity (GWa)"], 0) + hist_tra["value"] = hist_tra["value"] * share_oil_pipelines[trade_technology] + + hist_tra["year_act"] = hist_tra["YEAR"].astype(int) + hist_tra["unit"] = "GWa" + hist_tra["mode"] = "M1" + hist_tra["time"] = "year" + hist_tra = hist_tra[ + ["node_loc", "technology", "year_act", "value", "unit", "mode", "time"] + ] + hist_tra["year_act"] = hist_tra["year_act"].astype(int) + hist_tra.to_csv( + os.path.join(gem_dir_out, "historical_activity_trade_GEM.csv"), index=False + ) + hist_tra.to_csv(os.path.join(trade_dir, "historical_activity.csv"), index=False) + hist_tra.to_csv( + os.path.join(trade_dir_out, "historical_activity.csv"), index=False + ) + + # Historical new capacity + hist_cap = df[["EXPORTER", "IMPORTER", "YEAR", "LengthMergedKm"]] + hist_cap = hist_cap.rename(columns={"LengthMergedKm": "CAPACITY_KM"}) + + hist_cap = ( + hist_cap.groupby(["EXPORTER", "IMPORTER", "YEAR"])["CAPACITY_KM"] + .sum() + .reset_index() + ) + hist_cap = hist_cap.sort_values(by=["EXPORTER", "IMPORTER", "YEAR"], ascending=True) + hist_cap = hist_cap[ + (hist_cap["YEAR"] < first_model_year) & (hist_cap["YEAR"] > 1999) + ] + + hist_cap = hist_cap.sort_values( + by=["EXPORTER", "IMPORTER", "YEAR"], ascending=False + ) + hist_cap = hist_cap[["EXPORTER", "IMPORTER", "YEAR", "CAPACITY_KM"]] + + hist_cap["CAPACITY_KM"] = hist_cap["CAPACITY_KM"] / 5 # Duration time is 5 years + + hist_cap["node_loc"] = hist_cap["EXPORTER"] + hist_cap["technology"] = ( + flow_technology + "_" + hist_cap["IMPORTER"].str.lower().str.split("_").str[-1] + ) + hist_cap["value"] = round(hist_cap["CAPACITY_KM"], 0) + hist_cap["YEAR"] = hist_cap["YEAR"].astype(int) + hist_cap = message_ix.make_df( + "historical_new_capacity", + node_loc=hist_cap["node_loc"], + technology=hist_cap["technology"], + year_vtg=hist_cap["YEAR"], + value=hist_cap["value"], + unit="km", + ) + hist_cap.to_csv( + os.path.join(gem_dir_out, "historical_new_capacity_GEM.csv"), index=False + ) + hist_cap.to_csv(os.path.join(flow_dir, "historical_new_capacity.csv"), index=False) + hist_cap.to_csv( + os.path.join(flow_dir_out, "historical_new_capacity.csv"), index=False + ) + + # Input + inputdf = ( + df.groupby(["EXPORTER", "IMPORTER"])[["Capacity (GWa)", "LengthMergedKm"]] + .sum() + .reset_index() + ) + inputdf["Capacity (GWa/km)"] = inputdf["Capacity (GWa)"] / inputdf["LengthMergedKm"] + inputdf = inputdf[["EXPORTER", "IMPORTER", "Capacity (GWa/km)"]].drop_duplicates() + inputdf["node_loc"] = inputdf["EXPORTER"] + inputdf["technology"] = ( + trade_technology + + "_exp_" + + inputdf["IMPORTER"].str.lower().str.split("_").str[-1] + ) + inputdf["value_update"] = round((1 / inputdf["Capacity (GWa/km)"]), 0) + inputdf["commodity"] = ( + flow_commodity + "_" + inputdf["IMPORTER"].str.lower().str.split("_").str[-1] + ) + inputdf = inputdf[["node_loc", "technology", "value_update", "commodity"]] # km/GWa + + basedf = pd.read_csv(os.path.join(trade_dir, "input.csv")) + # The largest capacity pipelines have maximum 300,000GWh (~30bcm) annually + basedf["value"] = np.where( + basedf["commodity"].str.contains(flow_commodity), 30, basedf["value"] + ) + + inputdf = basedf.merge( + inputdf, + left_on=["node_loc", "technology", "commodity"], + right_on=["node_loc", "technology", "commodity"], + how="left", + ) + inputdf["value"] = np.where( + (~inputdf["value_update"].isnull()) + & (inputdf["value_update"] < 10000) + & (inputdf["commodity"].str.contains(flow_commodity)), + inputdf["value_update"], + inputdf["value"], + ) + inputdf = inputdf.drop(["value_update"], axis=1) + + inputdf.to_csv(os.path.join(gem_dir_out, "inputs_GEM.csv"), index=False) + inputdf.to_csv(os.path.join(trade_dir, "input.csv"), index=False) + inputdf.to_csv(os.path.join(trade_dir_out, "input.csv"), index=False) diff --git a/message_ix_models/tools/bilateralize/utils.py b/message_ix_models/tools/bilateralize/utils.py new file mode 100644 index 0000000000..38e394c7a6 --- /dev/null +++ b/message_ix_models/tools/bilateralize/utils.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +""" +Utility functions for the bilateralize tool +""" + +# Import packages +import logging +import os +import sys + +import yaml + +from message_ix_models.util import package_data_path + + +def get_logger(name: str): + # Set the logging level to INFO (will show INFO and above messages) + log = logging.getLogger(name) + log.setLevel(logging.INFO) + + # Define the format of log messages: + handler = logging.StreamHandler(sys.stdout) + formatter = logging.Formatter("%(name)s %(asctime)s %(levelname)s %(message)s") + + # Apply the format to the handler + handler.setFormatter(formatter) + + # Add the handler to the logger + log.addHandler(handler) + + return log + + +# %% Load config yaml +def load_config( + project_name: str | None = None, + config_name: str | None = None, + load_tec_config: bool = False, +): + """ + Load config file and optional trade-specific config files. + + Args: + project_name: Name of the project (message_ix_models/project/[THIS]) + config_name: Name of the base config file (e.g., config.yaml) + load_tec_config: If True, load the trade-specific config files + + Returns: + config: Config dictionary (base config) + config_path: Path to the config file + tec_config_dict: Dictionary of trade-specific config file + """ + # Load config + if project_name is None: + config_path = os.path.abspath( + os.path.join( + os.path.dirname(package_data_path("bilateralize")), + "bilateralize", + "configs", + "base_config.yaml", + ) + ) + if project_name is not None: + if config_name is None: + config_name = "config.yaml" + config_path = os.path.abspath( + os.path.join( + os.path.dirname(package_data_path(project_name)), + os.path.pardir, + "project", + project_name, + config_name, + ) + ) + + with open(config_path, "r") as f: + config = yaml.safe_load(f) # safe_load is recommended over load for security + + if not load_tec_config: + return config, config_path + else: + tec_config_dict = {} + for tec in config["covered_trade_technologies"]: + tec_config_path = os.path.abspath( + os.path.join( + os.path.dirname(package_data_path("bilateralize")), + "bilateralize", + "configs", + tec + ".yaml", + ) + ) + with open(tec_config_path, "r") as f: + tec_config = yaml.safe_load(f) + tec_config_dict[tec] = tec_config + return config, config_path, tec_config_dict diff --git a/message_ix_models/tools/bilateralize/workflow.py b/message_ix_models/tools/bilateralize/workflow.py new file mode 100644 index 0000000000..249d12b359 --- /dev/null +++ b/message_ix_models/tools/bilateralize/workflow.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +""" +Workflow for bilateralizing trade flows in MESSAGEix + +This includes: +1. Prepare edit files for bilateralization +2. Move data from bare files to a dictionary to update a MESSAGEix scenario +3. Update MESSAGEix scenario(s) with bilateralized dictionary and + solve scenario using the ixmp database or save as a GDX data + file for direct solve in GAMS. +""" + +# Import packages +from message_ix_models.tools.bilateralize.bare_to_scenario import bare_to_scenario +from message_ix_models.tools.bilateralize.load_and_solve import load_and_solve +from message_ix_models.tools.bilateralize.prepare_edit import prepare_edit_files + +# Project setup +project_name = None # Name of the project (e.g., 'newpathways') +config_name = None # Name of the config file (e.g., 'config.yaml') + +# Prepare edit files for bilateralization +prepare_edit_files(project_name=project_name, config_name=config_name) + +# Move data from bare files to a dictionary to update a MESSAGEix scenario +trade_dict = bare_to_scenario(project_name=project_name, config_name=config_name) + +# Update MESSAGEix scenario(s) with bilateralized dictionary and solve scenario +load_and_solve( + project_name=project_name, + config_name=config_name, + scenario=None, # Specifies MESSAGEix scenario, or will use project yaml + trade_dict=trade_dict, + solve=True, +) diff --git a/pyproject.toml b/pyproject.toml index e97dc643e0..55e9b34528 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,7 @@ Repository = "https://github.com/iiasa/message-ix-models" "Issue tracker" = "https://github.com/iiasa/message-ix-models/issues" [project.optional-dependencies] +bilateralize = ["scgraph"] buildings = ["jaydebeapi"] docs = [ # sphinx.ext.autosummary covers the test suite, so all test requirements @@ -73,7 +74,7 @@ migrate = [ report = ["plotnine", "xlsxwriter"] tests = [ "message-ix[tests]", - "message_ix_models[buildings,material,report,transport]", + "message_ix_models[bilateralize,buildings,material,report,transport]", "pytest", "pytest-cov", "pytest-timeout", @@ -108,6 +109,7 @@ omit = [ # The single existing test is far from enough "message_ix_models/report/legacy/*", "message_ix_models/util/migrate.py", + "message_ix_models/tools/bilateralize/workflow.py", # This is a workflow example ] [tool.mypy]