diff --git a/CHANGELOG.md b/CHANGELOG.md index 15e0c0b..d44f251 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ # Changelog + All notable changes to this project will be documented in this file. The format is inpired from [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) @@ -26,119 +27,163 @@ Template: ### Removed - basic description [#PR/#Issue/#Commit] ``` + ## [Unreleased] - 20XX-MM-DD ### Added +- + ### Changed +- + ### Removed ______________________________________________________________________ + +## [v0.3.3] - 2024-04-15 + +### Added + +- New settings module [#44](https://github.com/OpenEnergyPlatform/oem2orm/pull/44) + +### Changed + +- Improve logging and error messages [table creation, data upload, metadata registration](#38) +- Fix error if field type is None +- Update oep host url [#44](https://github.com/OpenEnergyPlatform/oem2orm/pull/44) + +______________________________________________________________________ + ## [0.3.2] - 2022-11-29 ### Added + - token can be passed as parameter (to support usage in APIs) ### Changed + - Error is raised if schema does not exist - Metadata compilance checks now inculue optinal jsonschema validation for metadata (PR#32) - MetadataError is thrown if uploading metadata to OEP fails ______________________________________________________________________ + ## [0.3.1] - 2022-10-24 ### Changed + - fix module not installed error after pip install and import of oem2orm PR(#26) ______________________________________________________________________ + ## [0.3.0] - 2022-10-24 ### Added + - Option to create tables from OEM JSON instead of file - New module to check if metadata is oep compliant. Can check (omi's 1 parse 2 compile) oemetadata v1.5 and v1.4 (PR#23) - Add PYPI release workflow to automate python package releases for pypi test and pypi official (PR#22) ### Changed + - omi_validateMD was outdated and now runs the new oep compliance checks. ______________________________________________________________________ + ## [0.2.7] - 2021-03-11 ### Added + - enable console usage [PR#8] -- new package requirement "omi" -- support for sqlachemy "numeric" data type +- new package requirement "omi" +- support for sqlachemy "numeric" data type ### Changed + - fix missing dependency that made pip install fail [ISSUE#1;PR#8] ### Removed -- console script, remove table delete + +- console script, remove table delete ______________________________________________________________________ + ## [0.2.6] - 2020-09-23 ### Added + - support for datatypes "hstore" and "decimal" - provide new example files that work with oem2orm ______________________________________________________________________ + ## [0.2.5] - 2020-08-06 ### Changed + - fix installation error caused by jmespath package dependency ______________________________________________________________________ + ## [0.2.4] - 2020-07-20 ### Added + - Support to setup the OEP API-URL - Metadata Up- and download are supported - Save downloaded metadata to file - Validate metadata using OMI parser v1.4.0 ### Changed + - change functions names ______________________________________________________________________ + ## [0.2.3] - 2020-06-02 ### Added + - provide a minimal working example as jupyter notebook tutorial -- New OEP-API related functions: Prepare the oemetadata string to send to api +- New OEP-API related functions: Prepare the oemetadata string to send to api - Simple User Input function to set the OEP-API-Token ### Changed + - Update README - include OEP public schema (whitelist) check - Spatial types from Geoalchemy2 do not set a spatial_index anymore ______________________________________________________________________ + ## [0.2.2] - 2020-06-02 ### Added + - new function: setting up a logger ### Changed + - add missing input parameter - extended description in changelog -- Fix logging +- Fix logging ______________________________________________________________________ + ## [0.2.0] - 2020-05-27 ### Added + - new function: delete tables from DB now possible -- new function: select the oem data folder +- new function: select the oem data folder - new function: tables are collected and ordered by fk (increase usability) ### Changed + - added docstrings ### Removed -- the user is no longer required to use a for loop in the main function to collect tables - - - +- the user is no longer required to use a for loop in the main function to collect tables diff --git a/README.md b/README.md index 50e3ec0..e23f72b 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ see [Pipx-Documentation](https://pypa.github.io/pipx/) for further information. ## Usage: -This tool is part of the open-energy-metadata (OEM) integration into the [OEP](https://openenergy-platform.org/). +This tool is part of the open-energy-metadata (OEM) integration into the [OEP](https://openenergyplatform.org/). To use this tool with the OEP API you need to be signed up to the OEP since you need to provide an API-Token. diff --git a/oem2orm/oep_oedialect_oem2orm.py b/oem2orm/oep_oedialect_oem2orm.py index 44e9b08..ac58353 100644 --- a/oem2orm/oep_oedialect_oem2orm.py +++ b/oem2orm/oep_oedialect_oem2orm.py @@ -19,6 +19,7 @@ from oem2orm.postgresql_types import TYPES from oem2orm.oep_compliance import run_metadata_checks +from oem2orm.settings import OEP_URL, OEP_API_URL # prepare connection string to connect via oep API CONNECTION_STRING = "{engine}://{user}:{token}@{host}" @@ -38,14 +39,14 @@ class MetadataError(Exception): pass -def setup_logger(): +def setup_logger(logger_level: str = "Yes"): """ Easy logging setup depending on user input. Provides a logger for INFO level logging. :return: logging.INFO or none """ - logger_level = input("Display logging information[Yes] or [No]:") + if re.fullmatch("[Yy]es", logger_level): - print("logging activated") + print("Logging activated") return logging.basicConfig( format="%(levelname)s:%(message)s", level=logging.INFO ) @@ -53,7 +54,7 @@ def setup_logger(): pass -def setup_db_connection(engine="postgresql+oedialect", host="openenergy-platform.org"): +def setup_db_connection(engine="postgresql+oedialect", host=OEP_URL): """ Create SQLAlchemy connection to Database API with Username and Token. Default is the OEP RESTful-API. @@ -81,9 +82,8 @@ def setup_db_connection(engine="postgresql+oedialect", host="openenergy-platform def setupApiAction(schema, table, token=None): API_ACTION = namedtuple("API_Action", ["dest_url", "headers"]) - OEP_URL = "https://openenergy-platform.org" - url = OEP_URL + "/api/v0/schema/{schema}/tables/{table}/meta/".format( + url = OEP_API_URL + "schema/{schema}/tables/{table}/meta/".format( schema=schema, table=table ) @@ -107,8 +107,10 @@ def create_tables(db: DB, tables: List[sa.Table]): :return: none """ for table in tables: + logging.info(f"Working on table: {table}") if not db.engine.dialect.has_schema(db.engine, table.schema): - error_msg = f'The provided database schema: "{table.schema}" does not exist. Please use an existing schema' + error_msg = f'The provided database schema: "{table.schema}" does not exist. Please use an existing ' \ + f'schema from the `name` column from: {OEP_URL}/dataedit/schemas' logging.info(error_msg) raise DatabaseError(error_msg) else: @@ -117,7 +119,7 @@ def create_tables(db: DB, tables: List[sa.Table]): table.create(checkfirst=True) logging.info(f"Created table {table.name}") except oedialect.engine.ConnectionException as ce: - error_msg = f'Error when uploading table "{table.name}".' + error_msg = f'Error when uploading table "{table.name}". Reason: {ce}.' logging.error(error_msg) raise DatabaseError(error_msg) from ce except sa.exc.ProgrammingError as pe: @@ -203,7 +205,7 @@ def create_tables_from_metadata_file( # Get column type: try: column_type = TYPES[field["type"]] - except KeyError: + except (KeyError, ValueError): raise MetadataError( "Unknown column type", field, field["type"], metadata_file ) @@ -268,7 +270,7 @@ def check_oep_api_schema_whitelist(oem_schema): return True else: logging.info( - "The OEP-API does not allow to write un-reviewed data to another schema then model_draft or sandbox" + "The OEP-API does not allow to write un-reviewed data to another schema then 'model_draft' or 'sandbox'" ) return False @@ -314,7 +316,7 @@ def collect_tables_from_oem(db: DB, oem_folder_path): def load_json(filepath): - logging.info("reading %s" % filepath) + logging.info("Reading metadata: %s" % filepath) with open(filepath, "rb") as f: return json.load(f) @@ -373,18 +375,26 @@ def api_updateMdOnTable(metadata, token=None): schema = getTableSchemaNameFromOEM(metadata)[0] table = getTableSchemaNameFromOEM(metadata)[1] - logging.info("UPDATE METADATA") + logging.info(f"Update metadata on table: {table}") api_action = setupApiAction(schema, table, token) resp = requests.post(api_action.dest_url, json=metadata, headers=api_action.headers) if resp.status_code == 200: - logging.info(" ok.") - logging.info(api_action.dest_url) + logging.info(f"METADATA SUCCESSFULLY UPDATED: {table}") + logging.info(f"Link to updated metadata on OEP: {api_action.dest_url}") else: - error_msg = resp.json() - logging.info(error_msg) - logging.info("HTTP status code: ") - logging.info(resp.status_code) - raise MetadataError(f"Uploading of metadata failed. Response from OEP: {error_msg}") + oep_err_msg_header = re.search("

(.*)

", resp.text).group(1) + err_message = ( + f"Uploading of metadata failed. HTTPS response from OEP: {resp.status_code}, Message: {oep_err_msg_header}, URL: {resp.url}" + "" + ) + if not resp.text.startswith("{"): + raise MetadataError( + f"The response text doesn't seem to be a json: {resp.text[:40]}..... \n\n Please check " + f"that the table is already created on the OEP! \n {err_message}" + ) + else: + # in case a json is returned and there is a json error + raise MetadataError(err_message, resp.json()) def api_downloadMd(schema, table, token=None): diff --git a/oem2orm/postgresql_types.py b/oem2orm/postgresql_types.py index 78c00af..fca5dcd 100644 --- a/oem2orm/postgresql_types.py +++ b/oem2orm/postgresql_types.py @@ -40,6 +40,8 @@ class DatabaseTypes: } def __getitem__(self, item): + if item is None: + raise ValueError("Field type is 'None'.") if item.split(" ")[-1] == "array": db_type = self.types[item[:-6]] return psql.ARRAY(db_type) diff --git a/oem2orm/requirements.txt b/oem2orm/requirements.txt index aecc812..b1db214 100644 --- a/oem2orm/requirements.txt +++ b/oem2orm/requirements.txt @@ -2,7 +2,7 @@ sqlalchemy psycopg2-binary jmespath GeoAlchemy2 -oedialect +oedialect==0.1.1 geopandas pandas omi \ No newline at end of file diff --git a/oem2orm/settings.py b/oem2orm/settings.py new file mode 100644 index 0000000..0154bfc --- /dev/null +++ b/oem2orm/settings.py @@ -0,0 +1,2 @@ +OEP_URL = "openenergy-platform.org" +OEP_API_URL = "https://openenergy-platform.org/api/v0/" diff --git a/setup.py b/setup.py index 4b5aefa..436d026 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="oem2orm", - version="0.3.2", + version="0.3.3", author="henhuy, jh-RLI", author_email="Hendrik.Huyskens@rl-institut.de", description="SQLAlchemy module to generate ORM, read from data model (oedatamodel) in open-energy-metadata JSON format", @@ -28,7 +28,7 @@ "Operating System :: OS Independent", ], python_requires='>=3.6', - install_requires=['sqlalchemy==1.3.14', 'oedialect', 'requests', 'jmespath', 'omi', 'click'], # Optional + install_requires=['sqlalchemy==1.3.14', 'oedialect==1.1', 'requests', 'jmespath', 'omi', 'click'], # Optional project_urls={ # Optional 'Bug Reports': 'https://github.com/OpenEnergyPlatform/oem2orm/issues', 'Source': 'https://github.com/OpenEnergyPlatform/oem2orm/tree/develop/oem2orm',