Update geo and configs (#7740)

* update geo

* update the MX zone config

* update the VN zone configs

* merge Hawaii geo

* Fix Hawaii configs

* geo tweaks

* fix geofile

* add KY config

* add MO config

* add MC config

* add BM config

* add SC config

* add MV config

* add CW config

* prettier

* adjust exchange configs

* fix CENANCE parser

* fix CENANCE parser

* adjust validator to check for valid zones in retired zones as well

* format

* fix ENTE parser

config/exchanges/GT_MX.yaml

@@ -1,4 +1,6 @@
 lonlat:
   - -90.967
   - 16.035
+parsers:
+  exchange: ENTE.fetch_exchange
 rotation: -55

config/zones/BM.yaml

@@ -0,0 +1,10 @@
+bounding_box:
+  - - -65.3842596183096
+    - 31.747150655733627
+  - - -64.1476046795164
+    - 32.88957682629426
+contributors:
+  - VIKTORVAV99
+country: BM
+region: Americas
+timezone: Atlantic/Bermuda

config/zones/CW.yaml

@@ -0,0 +1,10 @@
+bounding_box:
+  - - -69.65514614798195
+    - 11.537431473445437
+  - - -68.23200926588031
+    - 12.889546858796207
+contributors:
+  - VIKTORVAV99
+country: CW
+region: Americas
+timezone: America/Curacao

config/zones/KY.yaml

@@ -0,0 +1,10 @@
+bounding_box:
+  - - -81.92729962458039
+    - 18.761071915595686
+  - - -79.22230689375876
+    - 20.255142692881883
+contributors:
+  - VIKTORVAV99
+country: KY
+region: Americas
+timezone: America/Cayman

config/zones/MC.yaml

@@ -0,0 +1,10 @@
+bounding_box:
+  - - 6.908986027133928
+    - 43.22474953690665
+  - - 7.939894931523015
+    - 44.25193523970334
+contributors:
+  - VIKTORVAV99
+country: MC
+region: Europe
+timezone: Europe/Monaco

config/zones/MO.yaml

@@ -0,0 +1,10 @@
+bounding_box:
+  - - 113.02779759582114
+    - 21.57645485902869
+  - - 114.13042179357811
+    - 22.71674764489182
+contributors:
+  - VIKTORVAV99
+country: MO
+region: Asia
+timezone: Asia/Macau

config/zones/MV.yaml

@@ -0,0 +1,10 @@
+bounding_box:
+  - - 72.57531291255368
+    - -1.2023642515857569
+  - - 74.0538126436328
+    - 4.74016710592131
+contributors:
+  - VIKTORVAV99
+country: MV
+region: Asia
+timezone: Indian/Maldives

config/zones/MX.yaml

@@ -83,14 +83,4 @@ region: Americas
 sources:
   IEA, Electricity generation mix in Mexico, 1 Jan - 30 Sep, 2019 and 2020, IEA, Paris:
     link: https://www.iea.org/data-and-statistics/charts/electricity-generation-mix-in-mexico-1-jan-30-sep-2019-and-2020
-subZoneNames:
-  - MX-BC
-  - MX-BCS
-  - MX-CE
-  - MX-NE
-  - MX-NO
-  - MX-NW
-  - MX-OC
-  - MX-OR
-  - MX-PN
 timezone: America/Mexico_City

config/zones/SC.yaml

@@ -0,0 +1,10 @@
+bounding_box:
+  - - 54.70826687574969
+    - -5.302634091088595
+  - - 56.356329489439965
+    - -3.7757178049977007
+contributors:
+  - VIKTORVAV99
+country: SC
+region: Africa
+timezone: Indian/Mahe

config/zones/US-HI.yaml

@@ -0,0 +1,3 @@
+country: US
+region: Americas
+timezone: Pacific/Honolulu

electricitymap/contrib/config/__init__.py

@@ -21,6 +21,7 @@
 CONFIG_DIR = Path(__file__).parent.parent.parent.parent.joinpath("config").resolve()
 
 ZONES_CONFIG = read_zones_config(CONFIG_DIR)
+RETIRED_ZONES_CONFIG = read_zones_config(CONFIG_DIR, retired=True)
 EXCHANGES_CONFIG = read_exchanges_config(CONFIG_DIR)
 
 EU_ZONES = [

electricitymap/contrib/config/reading.py

@@ -15,10 +15,12 @@ def read_defaults(config_dir) -> dict[str, Any]:
         return yaml.load(file)
 
 
-def read_zones_config(config_dir) -> dict[ZoneKey, Any]:
+def read_zones_config(config_dir, retired=False) -> dict[ZoneKey, Any]:
     """Reads all the zone config files."""
     zones_config: dict[ZoneKey, Any] = {}
-    for zone_path in config_dir.joinpath("zones").glob("*.yaml"):
+    for zone_path in config_dir.joinpath(
+        "retired_zones" if retired is True else "zones"
+    ).glob("*.yaml"):
         zone_key = ZoneKey(zone_path.stem)
         with open(zone_path, encoding="utf-8") as file:
             zones_config[zone_key] = yaml.load(file)

electricitymap/contrib/lib/models/events.py

@@ -11,7 +11,11 @@
 import pandas as pd
 from pydantic import BaseModel, PrivateAttr, ValidationError, validator
 
-from electricitymap.contrib.config import EXCHANGES_CONFIG, ZONES_CONFIG
+from electricitymap.contrib.config import (
+    EXCHANGES_CONFIG,
+    RETIRED_ZONES_CONFIG,
+    ZONES_CONFIG,
+)
 from electricitymap.contrib.config.constants import PRODUCTION_MODES, STORAGE_MODES
 from electricitymap.contrib.lib.models.constants import VALID_CURRENCIES
 from electricitymap.contrib.lib.types import ZoneKey
@@ -306,7 +310,7 @@ class Event(BaseModel, ABC):
 
     @validator("zoneKey")
     def _validate_zone_key(cls, v):
-        if v not in ZONES_CONFIG:
+        if v not in ZONES_CONFIG and v not in RETIRED_ZONES_CONFIG:
             raise ValueError(f"Unknown zone: {v}")
         return v
 

parsers/CENACE.py

@@ -12,7 +12,6 @@
 from dateutil import tz
 from requests import Response, Session
 
-from electricitymap.contrib.config import ZONES_CONFIG
 from electricitymap.contrib.lib.models.event_lists import (
     ExchangeList,
     ProductionBreakdownList,
@@ -38,9 +37,12 @@
     "MX-NE->MX-OC": "IntercambioNES-OCC",
     "MX-NO->MX-OC": "IntercambioNTE-OCC",
     "MX-NW->MX-OC": "IntercambioNOR-OCC",
+    "MX->US-CAL-CISO": "IntercambioUSA-BCA",
     "MX-BC->US-CAL-CISO": "IntercambioUSA-BCA",
+    "MX->US-TEX-ERCO": "DummyValueNotUsed",
     "MX-NO->US-TEX-ERCO": "IntercambioUSA-NTE",
     "MX-NE->US-TEX-ERCO": "IntercambioUSA-NES",
+    "BZ->MX": "IntercambioPEN-BEL",
     "BZ->MX-PN": "IntercambioPEN-BEL",
 }
 
@@ -222,16 +224,22 @@ def fetch_MX_exchange(sorted_zone_keys: ZoneKey, s: Session) -> float:
     """Finds current flow between two Mexican control areas."""
     req = s.get(MX_EXCHANGE_URL, headers={"User-Agent": "Mozilla/5.0"})
     soup = BeautifulSoup(req.text, "html.parser")
-    exchange_div = soup.find("div", attrs={"id": EXCHANGES[sorted_zone_keys]})
-    val = exchange_div.text
-
     # cenace html uses unicode hyphens instead of minus signs and , as thousand separator
     trantab = str.maketrans({chr(8208): chr(45), ",": ""})
-
-    val = val.translate(trantab)
-    flow = float(val)
-
-    if sorted_zone_keys in ["BZ->MX-PN", "MX-CE->MX-OR", "MX-CE->MX-OC"]:
+    if sorted_zone_keys == "MX->US-TEX-ERCO":
+        exchange_div1 = soup.find("div", attrs={"id": "IntercambioUSA-NTE"})
+        exchange_div2 = soup.find("div", attrs={"id": "IntercambioUSA-NES"})
+        val1, val2 = exchange_div1.text, exchange_div2.text
+        val1 = val1.translate(trantab)
+        val2 = val2.translate(trantab)
+        flow = float(val1) + float(val2)
+    else:
+        exchange_div = soup.find("div", attrs={"id": EXCHANGES[sorted_zone_keys]})
+        val = exchange_div.text
+        val = val.translate(trantab)
+        flow = float(val)
+
+    if sorted_zone_keys in ["BZ->MX", "BZ->MX-PN", "MX-CE->MX-OR", "MX-CE->MX-OC"]:
         # reversal needed for these zones due to EM ordering
         flow = -1 * flow
 
@@ -294,8 +302,9 @@ def fetch_consumption(
     if demand_td is None:
         raise ParserException("CENACE.py", "Could not find demand cell", zone_key)
     demand = float(demand_td.text.replace(",", ""))
-    timezone = ZONES_CONFIG[zone_key].get("timezone")
-    if timezone is None:
+    if zone_key == "MX-BC" or zone_key == "MX-BCS":
+        timezone = "America/Tijuana"
+    else:
         timezone = "America/Mexico_City"
     consumption_list = TotalConsumptionList(logger)
     consumption_list.append(
@@ -308,28 +317,32 @@ def fetch_consumption(
 
 
 if __name__ == "__main__":
-    print(fetch_production("MX", target_datetime=datetime(year=2019, month=7, day=1)))
+    print(
+        fetch_production(
+            ZoneKey("MX"), target_datetime=datetime(year=2019, month=7, day=1)
+        )
+    )
     print("fetch_exchange(MX-NO, MX-NW)")
-    print(fetch_exchange("MX-NO", "MX-NW"))
+    print(fetch_exchange(ZoneKey("MX-NO"), ZoneKey("MX-NW")))
     print("fetch_exchange(MX-OR, MX-PN)")
-    print(fetch_exchange("MX-OR", "MX-PN"))
+    print(fetch_exchange(ZoneKey("MX-OR"), ZoneKey("MX-PN")))
     print("fetch_exchange(MX-NE, MX-OC)")
-    print(fetch_exchange("MX-NE", "MX-OC"))
+    print(fetch_exchange(ZoneKey("MX-NE"), ZoneKey("MX-OC")))
     print("fetch_exchange(MX-NE, MX-NO)")
-    print(fetch_exchange("MX-NE", "MX-NO"))
+    print(fetch_exchange(ZoneKey("MX-NE"), ZoneKey("MX-NO")))
     print("fetch_exchange(MX-OC, MX-OR)")
-    print(fetch_exchange("MX-OC", "MX-OR"))
+    print(fetch_exchange(ZoneKey("MX-OC"), ZoneKey("MX-OR")))
     print("fetch_exchange(MX-NE, US-TEX-ERCO)")
-    print(fetch_exchange("MX-NE", "US-TEX-ERCO"))
+    print(fetch_exchange(ZoneKey("MX-NE"), ZoneKey("US-TEX-ERCO")))
     print("fetch_exchange(MX-CE, MX-OC)")
-    print(fetch_exchange("MX-CE", "MX-OC"))
+    print(fetch_exchange(ZoneKey("MX-CE"), ZoneKey("MX-OC")))
     print("fetch_exchange(MX-PN, BZ)")
-    print(fetch_exchange("MX-PN", "BZ"))
+    print(fetch_exchange(ZoneKey("MX-PN"), ZoneKey("BZ")))
     print("fetch_exchange(MX-NO, MX-OC)")
-    print(fetch_exchange("MX-NO", "MX-OC"))
+    print(fetch_exchange(ZoneKey("MX-NO"), ZoneKey("MX-OC")))
     print("fetch_exchange(MX-NO, US-TEX-ERCO)")
-    print(fetch_exchange("MX-NO", "US-TEX-ERCO"))
+    print(fetch_exchange(ZoneKey("MX-NO"), ZoneKey("US-TEX-ERCO")))
     print("fetch_exchange(MX-NE, MX-OR)")
-    print(fetch_exchange("MX-NE", "MX-OR"))
+    print(fetch_exchange(ZoneKey("MX-NE"), ZoneKey("MX-OR")))
     print("fetch_exchange(MX-CE, MX-OR)")
-    print(fetch_exchange("MX-CE", "MX-OR"))
+    print(fetch_exchange(ZoneKey("MX-CE"), ZoneKey("MX-OR")))

parsers/ENTE.py

@@ -25,6 +25,7 @@
 TIMEZONE = ZoneInfo("America/Tegucigalpa")
 
 JSON_MAPPING = {
+    "GT->MX": "2LBR.LT400.1FR2-2LBR-01A.-.MW",
     "GT->MX-OR": "2LBR.LT400.1FR2-2LBR-01A.-.MW",
     "GT->SV": "3SISTEMA.LT230.INTER_NET_GT.CMW.MW",
     "GT->HN": "4LEC.LT230.2FR4-4LEC-01B.-.MW",