From e716d2712264e7d831fbf6b8e07b0352acf06e28 Mon Sep 17 00:00:00 2001 From: "RL-INSTITUT\\hendrik.huyskens" Date: Thu, 29 Sep 2022 16:04:20 +0200 Subject: [PATCH 01/16] Added metadata upload in case of single table in OEM --- CHANGELOG.md | 1 + oem2orm/main.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d44f251..122cd79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ Template: ## [Unreleased] - 20XX-MM-DD ### Added +- Metadata upload in case of single table in OEM - diff --git a/oem2orm/main.py b/oem2orm/main.py index 0f62752..7028194 100644 --- a/oem2orm/main.py +++ b/oem2orm/main.py @@ -17,6 +17,10 @@ def create_tables(metadata_folder): folder = pathlib.Path.cwd() / metadata_folder tables = oep_oedialect_oem2orm.collect_tables_from_oem(db, folder) oep_oedialect_oem2orm.create_tables(db, tables) + if len(tables) == 1: + # Upload metadata for single table + metadata = oep_oedialect_oem2orm.mdToDict(metadata_folder) + oep_oedialect_oem2orm.api_updateMdOnTable(metadata) @cli.command() From a5eaa9479ae9bf3d987dba1a101a7a8e08875f69 Mon Sep 17 00:00:00 2001 From: "RL-INSTITUT\\hendrik.huyskens" Date: Fri, 14 Oct 2022 08:37:28 +0200 Subject: [PATCH 02/16] Dirty fix of primary key --- oem2orm/oep_oedialect_oem2orm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oem2orm/oep_oedialect_oem2orm.py b/oem2orm/oep_oedialect_oem2orm.py index ac58353..7c70e5e 100644 --- a/oem2orm/oep_oedialect_oem2orm.py +++ b/oem2orm/oep_oedialect_oem2orm.py @@ -225,7 +225,7 @@ def create_tables_from_metadata_file( column = sa.Column( field["name"], column_type, - primary_key=field["name"] in primary_keys, + # primary_key=field["name"] in primary_keys, # FIXME: Should be activated, see https://github.com/OpenEnergyPlatform/oedialect/issues/43 comment=field["description"], ) columns.append(column) From 5072622b61dedaf9529e4de8e380510d3581eae8 Mon Sep 17 00:00:00 2001 From: Hendrik Huyskens Date: Tue, 16 Apr 2024 14:15:30 +0200 Subject: [PATCH 03/16] Fix oedialect version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 436d026..db03824 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ "Operating System :: OS Independent", ], python_requires='>=3.6', - install_requires=['sqlalchemy==1.3.14', 'oedialect==1.1', 'requests', 'jmespath', 'omi', 'click'], # Optional + install_requires=['sqlalchemy==1.3.14', 'oedialect==0.1.1', 'requests', 'jmespath', 'omi', 'click'], # Optional project_urls={ # Optional 'Bug Reports': 'https://github.com/OpenEnergyPlatform/oem2orm/issues', 'Source': 'https://github.com/OpenEnergyPlatform/oem2orm/tree/develop/oem2orm', From c8923ed29429ca7235d0f2040799126796595765 Mon Sep 17 00:00:00 2001 From: Hendrik Huyskens Date: Tue, 16 Apr 2024 14:29:26 +0200 Subject: [PATCH 04/16] Update sqlalchemy version to v1.3.16 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index db03824..7f86f61 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ "Operating System :: OS Independent", ], python_requires='>=3.6', - install_requires=['sqlalchemy==1.3.14', 'oedialect==0.1.1', 'requests', 'jmespath', 'omi', 'click'], # Optional + install_requires=['sqlalchemy==1.3.16', 'oedialect==0.1.1', 'requests', 'jmespath', 'omi', 'click'], # Optional project_urls={ # Optional 'Bug Reports': 'https://github.com/OpenEnergyPlatform/oem2orm/issues', 'Source': 'https://github.com/OpenEnergyPlatform/oem2orm/tree/develop/oem2orm', From 35a55f375b9ce1ab2e89b85034f594c6d4004d95 Mon Sep 17 00:00:00 2001 From: Hendrik Huyskens Date: Tue, 16 Apr 2024 14:15:30 +0200 Subject: [PATCH 05/16] Fix oedialect version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 436d026..db03824 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ "Operating System :: OS Independent", ], python_requires='>=3.6', - install_requires=['sqlalchemy==1.3.14', 'oedialect==1.1', 'requests', 'jmespath', 'omi', 'click'], # Optional + install_requires=['sqlalchemy==1.3.14', 'oedialect==0.1.1', 'requests', 'jmespath', 'omi', 'click'], # Optional project_urls={ # Optional 'Bug Reports': 'https://github.com/OpenEnergyPlatform/oem2orm/issues', 'Source': 'https://github.com/OpenEnergyPlatform/oem2orm/tree/develop/oem2orm', From 00373a8cf2f0f80adb7d4b7e576c53bbf3cf48b8 Mon Sep 17 00:00:00 2001 From: Hendrik Huyskens Date: Tue, 16 Apr 2024 14:29:26 +0200 Subject: [PATCH 06/16] Update sqlalchemy version to v1.3.16 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index db03824..7f86f61 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ "Operating System :: OS Independent", ], python_requires='>=3.6', - install_requires=['sqlalchemy==1.3.14', 'oedialect==0.1.1', 'requests', 'jmespath', 'omi', 'click'], # Optional + install_requires=['sqlalchemy==1.3.16', 'oedialect==0.1.1', 'requests', 'jmespath', 'omi', 'click'], # Optional project_urls={ # Optional 'Bug Reports': 'https://github.com/OpenEnergyPlatform/oem2orm/issues', 'Source': 'https://github.com/OpenEnergyPlatform/oem2orm/tree/develop/oem2orm', From f65b9dbcf1e12dc4e7dcec27954c015923b57c03 Mon Sep 17 00:00:00 2001 From: Hendrik Huyskens Date: Tue, 16 Apr 2024 14:54:16 +0200 Subject: [PATCH 07/16] Update CHANGELOG.md entry for fix --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d44f251..30b8388 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,14 @@ ______________________________________________________________________ ## [v0.3.3] - 2024-04-15 +### Fixed +- oedialect version to v0.1.1 +- sqlalchemy version to v1.3.16 + +______________________________________________________________________ + +## [v0.3.3] - 2024-04-15 + ### Added - New settings module [#44](https://github.com/OpenEnergyPlatform/oem2orm/pull/44) From 5b2aa48753c7205fff4f7c1876c51ea7d4e686e0 Mon Sep 17 00:00:00 2001 From: Jonas Huber Date: Tue, 16 Apr 2024 17:07:05 +0200 Subject: [PATCH 08/16] format changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 122cd79..69db2ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ Template: ## [Unreleased] - 20XX-MM-DD ### Added + - Metadata upload in case of single table in OEM - From 21e7890b9ea778bd3de6c61e2dc732c8d0afe8aa Mon Sep 17 00:00:00 2001 From: Jonas Huber Date: Tue, 16 Apr 2024 17:09:12 +0200 Subject: [PATCH 09/16] activate primary_key=field["name"] and format file --- oem2orm/oep_oedialect_oem2orm.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/oem2orm/oep_oedialect_oem2orm.py b/oem2orm/oep_oedialect_oem2orm.py index 7c70e5e..dfaf450 100644 --- a/oem2orm/oep_oedialect_oem2orm.py +++ b/oem2orm/oep_oedialect_oem2orm.py @@ -109,8 +109,10 @@ def create_tables(db: DB, tables: List[sa.Table]): for table in tables: logging.info(f"Working on table: {table}") if not db.engine.dialect.has_schema(db.engine, table.schema): - error_msg = f'The provided database schema: "{table.schema}" does not exist. Please use an existing ' \ - f'schema from the `name` column from: {OEP_URL}/dataedit/schemas' + error_msg = ( + f'The provided database schema: "{table.schema}" does not exist. Please use an existing ' + f"schema from the `name` column from: {OEP_URL}/dataedit/schemas" + ) logging.info(error_msg) raise DatabaseError(error_msg) else: @@ -119,7 +121,9 @@ def create_tables(db: DB, tables: List[sa.Table]): table.create(checkfirst=True) logging.info(f"Created table {table.name}") except oedialect.engine.ConnectionException as ce: - error_msg = f'Error when uploading table "{table.name}". Reason: {ce}.' + error_msg = ( + f'Error when uploading table "{table.name}". Reason: {ce}.' + ) logging.error(error_msg) raise DatabaseError(error_msg) from ce except sa.exc.ProgrammingError as pe: @@ -225,7 +229,8 @@ def create_tables_from_metadata_file( column = sa.Column( field["name"], column_type, - # primary_key=field["name"] in primary_keys, # FIXME: Should be activated, see https://github.com/OpenEnergyPlatform/oedialect/issues/43 + primary_key=field["name"] + in primary_keys, # TODO: Should be fixed, see https://github.com/OpenEnergyPlatform/oedialect/issues/43 comment=field["description"], ) columns.append(column) From 8d5b0840ab412343306a5844718a8ef08b9f59dc Mon Sep 17 00:00:00 2001 From: Jonas Huber Date: Tue, 16 Apr 2024 18:15:19 +0200 Subject: [PATCH 10/16] enhance readme / documentation --- README.md | 122 ++++++++++++++++++++++++------------------------------ 1 file changed, 53 insertions(+), 69 deletions(-) diff --git a/README.md b/README.md index e23f72b..7ff2550 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -# OEM to ORM +# oem 2 orm -Create database tables (and schema) from oemetadata json file(s) +Create database tables (and schema) from oemetadata json file(s). This tool is part of the open-energy-metadata (OEM) integration into the [OEP](https://openenergyplatform.org/). -## Installation: +## Installation You can install pacakge using standard python installation: ` @@ -15,94 +15,78 @@ pipx install oem2orm ` see [Pipx-Documentation](https://pypa.github.io/pipx/) for further information. +## Usage -## Usage: +Read the Restrictions section and have a look at our [tutorial](./tutorial/USAGE.md) section to get more information about the usage of oem2orm either as code module or CLI tool. The tutorials also provide information how to validate your oemetadata files. -This tool is part of the open-energy-metadata (OEM) integration into the [OEP](https://openenergyplatform.org/). -To use this tool with the OEP API you need to be signed up to the OEP since -you need to provide an API-Token. +### Restrictions -If you want to upload OEM that was officially reviewed you must clone the -OEP data-preprocessing repository on [GitHub](https://github.com/OpenEnergyPlatform/data-preprocessing). -The data-review folder contains all of the successfully reviewed OEM files. +To use this tool with the OEP API you need to be signed up to the OEP since +you need to provide an API-Token. -For security reasons, tables can only be created in existing +For security reasons, tables can only be created in existing schemas and just in the schemas "model_draft" and "sandbox". -Keep in mind the current state is not fully tested. The code is -still quit error prone f.e. the postgres types (column datatype) are not fully +Keep in mind that f.e. the postgres types (column datatype) are not fully supported by the [oedialct](https://pypi.org/project/oedialect/) - work in progress. -### Terminal/CLI-Application -Step-by-Step: -0. pip and python have to be installed and setup on your machine -1. Create env from requirements.txt, and activate -2. Put the metadata file in the folder metadata or put your own folder in this - directory -3. execute the following in a terminal: -``` -pipx install oem2orm -oem2orm -Enter metadata folder name: -... -``` -4. Provide credentials and folder name in prompt -5. The table will be created +## Docs -### Import as Module +### Database connection -You can simply import this module in your Python script.py like this: +We use a global namedtuple called "DB" To store the sqlalchemy connection objects engine and metadata. +The namedtuple is available wen import oem2orm in a script. To establish the namedtuple use the function +setup_db_connection(). Now you can use DB.engine or DB.metadata. In the background the connection is established +using oedialect and the http API of the oeplatform website. -```python -from oem2orm import oep_oedialect_oem2orm as oem2orm -``` +### oem2orm generator -Now just call the functions provided in oem2orm like this: +The table objects (ORM) are generated on the fly from an oemetadata.json file. [oemetadata](https://github.com/OpenEnergyPlatform/oemetadata) is a metadata specification of the Open Energy Family. It includes about 50 fields that can be used to provide metadata for tabular data resources. +A subset of these fields are grouped in the key "resources" ([see out example](https://github.com/OpenEnergyPlatform/oemetadata/blob/develop/metadata/v160/example.json#L237-L388)) in the metadata. These fields describe the schema of +the data table (like table name, columns, data types & table relations). -Recommended execution order: -- Setup the logger -```python -oem2orm.setup_logger() -``` +The method oem2orm provides to create data tables on the OEP. It is especially useful if you attempt to automate the table creation and already use python or already have a oemetadata file available. The alternatives are: -- Setup the Database API connection as Namedtuple storing the SQLAlchemy engine and metadata: -```python -db = oem2orm.setup_db_connection() -``` +1. [manually describing](https://openenergyplatform.github.io/academy/tutorials/01_api/02_api_upload/#create-table) the table object in JSON and then use the oep HTTP API directly to create a table. +2. Use the [User Interface of the oeplatform website](https://openenergyplatform.org/dataedit/wizard/) to create a table and upload data. -- Provide the oem files in a folder (in the current directory). -- Pass the folder name to the function: -```python -metadata_folder = oem2orm.select_oem_dir(oem_folder_name="folder_name") -``` +### Oemetadata format -- Setup a SQLAlchemy ORM including all data-model in the provided oem files: -```python -orm = oem2orm.collect_ordered_tables_from_oem(db, metadata_folder) -``` +[Specification for the oemetadata](https://github.com/OpenEnergyPlatform/oemetadata) -- Create the tables on the Database: -```python -oem2orm.create_tables(db, orm) -``` +#### Oemetadata validation -- Delete all tables that have been created (all tables available in sa.metadata) -```python -oem2orm.delete_tables(db, orm) -``` +The oemetadata specification is integrated into the open energy platform using a tool called [omi (metadata integration)](https://github.com/OpenEnergyPlatform/omi). OMI provides functionality to run validation checks on the metadata up to the oemetadata version 1.6.0. oem2orm also provides a minimal oep compliance check that mocks the checks that are run on the oep website once the metadata is uploaded to a table. -## Docs: +#### Supported column data types -### Database connection -We use a global namedtuple called "DB" To store the sqlalchemy connection objects engine and metadata. -The namedtuple is available wen import oem2orm in a script. To establish the namedtuple use the function -setup_db_connection(). Now you can use DB.engine or DB.metadata. - -### oem2orm generator +Currently oem2orm supports -#### Supported datatypes + "bigint" + "int": + "integer" + "varchar" + "json" + "text" + "timestamp" + "interval" + "string" + "float" + "boolean" + "date" + "hstore" + "decimal" + "numeric" + "double precision" #### Spatial Types -We create columns with spatial datatypes using Geoalchemy2. + + "geometry point": Geometry("POINT", spatial_index=False), + "geom": Geometry("GEOMETRY", spatial_index=False), + "geometry": Geometry("GEOMETRY", spatial_index=False), + +We create columns with spatial datatypes using Geoalchemy2. ## Database support + +We only tested this tool with PostgreSQL & sqlalchemy version 1.3 From 456f8eb62da252c86b18440223830474fc33f671 Mon Sep 17 00:00:00 2001 From: Jonas Huber Date: Tue, 16 Apr 2024 18:15:43 +0200 Subject: [PATCH 11/16] create a usage description --- tutorial/USAGE.md | 82 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 tutorial/USAGE.md diff --git a/tutorial/USAGE.md b/tutorial/USAGE.md new file mode 100644 index 0000000..c947539 --- /dev/null +++ b/tutorial/USAGE.md @@ -0,0 +1,82 @@ +# Usage + +This section gets you started with using the oem2orm tool. + +## oemetadata compliance check + +It is mandatory for oem2orm that you provide a valid oemetadata as input. Otherwise the functionality can not be granted. To provide some tooling within oem2orm that helps you check your metadata JSON files you can use the module `oep_compliance.py`. + +Run the oemetadata check (currently only for oemetadata version up to 1.5.2): + + from oem2orm.oep_compliance import run_metadata_checks + # assuming your oemetadata file is in a directory called "data" + run_metadata_checks(oemetadata = None, oemetadata_path = "data/oemetadata.json", check_jsonschema = False) + +## Terminal/CLI-Application + +Step-by-Step: +0. pip and python have to be installed and setup on your machine + +1. Create env from requirements.txt, and activate +2. Put the metadata file in the folder metadata or put your own folder in this + directory +3. execute the following in a terminal: + +``` +pipx install oem2orm +oem2orm +Enter metadata folder name: +... +``` + +4. Provide credentials and folder name in prompt +5. The table will be created + +## Import as Module + +You can simply import this module in your Python script.py like this: + +```python +from oem2orm import oep_oedialect_oem2orm as oem2orm +``` + +Now just call the functions provided in oem2orm like this: + +Recommended execution order: + +- Setup the logger + +```python +oem2orm.setup_logger() +``` + +- Setup the Database API connection as Namedtuple storing the SQLAlchemy engine and metadata: + +```python +db = oem2orm.setup_db_connection() +``` + +- Provide the oem files in a folder (in the current directory). +- Pass the folder name to the function: + +```python +metadata_folder = oem2orm.select_oem_dir(oem_folder_name="folder_name") +``` + +- Setup a SQLAlchemy ORM including all data-model in the provided oem files: + +```python +orm = oem2orm.collect_ordered_tables_from_oem(db, metadata_folder) +``` + +- Create the tables on the Database: + +```python +oem2orm.create_tables(db, orm) +``` + +- Delete all tables that have been created (all tables available in sa.metadata) + +```python +oem2orm.delete_tables(db, orm) +``` From 1572b61268bfec8254ead421da19948db6e11570 Mon Sep 17 00:00:00 2001 From: Jonas Huber Date: Tue, 16 Apr 2024 18:15:59 +0200 Subject: [PATCH 12/16] formatting --- oem2orm/main.py | 7 +++---- oem2orm/oep_compliance.py | 18 +++++++++++++----- oem2orm/oep_oedialect_oem2orm.py | 10 +++++++--- oem2orm/postgresql_types.py | 17 +++++++---------- 4 files changed, 30 insertions(+), 22 deletions(-) diff --git a/oem2orm/main.py b/oem2orm/main.py index 0f62752..c6d6181 100644 --- a/oem2orm/main.py +++ b/oem2orm/main.py @@ -1,4 +1,3 @@ - import click import pathlib @@ -11,7 +10,7 @@ def cli(): @cli.command() -@click.argument('metadata-folder', type=click.Path(exists=True)) +@click.argument("metadata-folder", type=click.Path(exists=True)) def create_tables(metadata_folder): db = oep_oedialect_oem2orm.setup_db_connection() folder = pathlib.Path.cwd() / metadata_folder @@ -20,7 +19,7 @@ def create_tables(metadata_folder): @cli.command() -@click.argument('metadata-folder', type=click.Path(exists=True)) +@click.argument("metadata-folder", type=click.Path(exists=True)) def delete_tables(metadata_folder): db = oep_oedialect_oem2orm.setup_db_connection() folder = pathlib.Path.cwd() / metadata_folder @@ -28,5 +27,5 @@ def delete_tables(metadata_folder): oep_oedialect_oem2orm.delete_tables(db, tables) -if __name__ == '__main__': +if __name__ == "__main__": cli() diff --git a/oem2orm/oep_compliance.py b/oem2orm/oep_compliance.py index 69bbcf4..d4d4a25 100644 --- a/oem2orm/oep_compliance.py +++ b/oem2orm/oep_compliance.py @@ -6,6 +6,7 @@ """ + import logging import pathlib import json @@ -24,6 +25,7 @@ logger = logging.getLogger() + def read_input_json(file_path: pathlib.Path = "tests/data/metadata_v15.json"): with open(file_path, "r", encoding="utf-8") as f: jsn = json.load(f) @@ -123,7 +125,9 @@ def check_oemetadata_is_oep_compatible(metadata): # ------------------------------------------- -def run_metadata_checks(oemetadata: dict = None, oemetadata_path: str = None, check_jsonschema: bool = False): +def run_metadata_checks( + oemetadata: dict = None, oemetadata_path: str = None, check_jsonschema: bool = False +): """ Runs metadata checks includes: - basic oep compliant check - tested by using omi's parsing and compiling @@ -165,10 +169,12 @@ def run_metadata_checks(oemetadata: dict = None, oemetadata_path: str = None, c schema = parser_validation.get_schema_by_metadata_version(metadata=metadata) result = parser_validation.is_valid(inp=metadata, schema=schema) if result is False: - result = result, parser_validation.validate(metadata=metadata, save_report=True) - + result = result, parser_validation.validate( + metadata=metadata, save_report=True + ) + return result - + if __name__ == "__main__": correct_v15_test_data = "tests/data/metadata_v15.json" @@ -177,7 +183,9 @@ def run_metadata_checks(oemetadata: dict = None, oemetadata_path: str = None, c meta = read_input_json(file_path=correct_v15_test_data) print("Check v15 metadata from file!") - result = run_metadata_checks(oemetadata_path=correct_v15_test_data, check_jsonschema=True) + result = run_metadata_checks( + oemetadata_path=correct_v15_test_data, check_jsonschema=True + ) print("Check v15 metadata from object!") run_metadata_checks(oemetadata=meta) diff --git a/oem2orm/oep_oedialect_oem2orm.py b/oem2orm/oep_oedialect_oem2orm.py index ac58353..64cf0bb 100644 --- a/oem2orm/oep_oedialect_oem2orm.py +++ b/oem2orm/oep_oedialect_oem2orm.py @@ -109,8 +109,10 @@ def create_tables(db: DB, tables: List[sa.Table]): for table in tables: logging.info(f"Working on table: {table}") if not db.engine.dialect.has_schema(db.engine, table.schema): - error_msg = f'The provided database schema: "{table.schema}" does not exist. Please use an existing ' \ - f'schema from the `name` column from: {OEP_URL}/dataedit/schemas' + error_msg = ( + f'The provided database schema: "{table.schema}" does not exist. Please use an existing ' + f"schema from the `name` column from: {OEP_URL}/dataedit/schemas" + ) logging.info(error_msg) raise DatabaseError(error_msg) else: @@ -119,7 +121,9 @@ def create_tables(db: DB, tables: List[sa.Table]): table.create(checkfirst=True) logging.info(f"Created table {table.name}") except oedialect.engine.ConnectionException as ce: - error_msg = f'Error when uploading table "{table.name}". Reason: {ce}.' + error_msg = ( + f'Error when uploading table "{table.name}". Reason: {ce}.' + ) logging.error(error_msg) raise DatabaseError(error_msg) from ce except sa.exc.ProgrammingError as pe: diff --git a/oem2orm/postgresql_types.py b/oem2orm/postgresql_types.py index fca5dcd..7987034 100644 --- a/oem2orm/postgresql_types.py +++ b/oem2orm/postgresql_types.py @@ -1,7 +1,7 @@ __copyright__ = "Reiner Lemoine Institut" -__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" -__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" -__author__ = "henhuy" +__license__ = "GNU Affero General Public License Version 3 (AGPL-3.0)" +__url__ = "https://github.com/openego/data_processing/blob/master/LICENSE" +__author__ = "henhuy" import sqlalchemy as sa import sqlalchemy.dialects.postgresql as psql @@ -27,16 +27,13 @@ class DatabaseTypes: "hstore": HSTORE, "decimal": sa.DECIMAL, "numeric": sa.NUMERIC, - # Spatial types - "geometry point": Geometry("POINT", spatial_index=False), - "geom": Geometry("GEOMETRY", spatial_index=False), - "geometry": Geometry("GEOMETRY", spatial_index=False), - + "geometry point": Geometry("POINT", spatial_index=False), + "geom": Geometry("GEOMETRY", spatial_index=False), + "geometry": Geometry("GEOMETRY", spatial_index=False), # not support with oedialect - "double precision": psql.DOUBLE_PRECISION + "double precision": psql.DOUBLE_PRECISION, # "double precision array": sa.ARRAY("DOUBLE_PRECISION"), - } def __getitem__(self, item): From 7923f8ab9eeddf6b208963cae58770c37d63b833 Mon Sep 17 00:00:00 2001 From: Jonas Huber Date: Tue, 16 Apr 2024 18:16:31 +0200 Subject: [PATCH 13/16] create new dir tutorials & move all example data there as well as the usage description --- .../OEDataModel-normalization-datapackage.json | 0 {oemetadata-upload => tutorial/data/oemetadata-upload}/flags.json | 0 .../data/oemetadata-upload}/locations.json | 0 .../data/oemetadata-upload}/series.json | 0 .../data/oemetadata-upload}/timespans.json | 0 .../data/oemetadata-upload}/variables.json | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename {datapackage_example => tutorial/data/datapackage_example}/OEDataModel-normalization-datapackage.json (100%) rename {oemetadata-upload => tutorial/data/oemetadata-upload}/flags.json (100%) rename {oemetadata-upload => tutorial/data/oemetadata-upload}/locations.json (100%) rename {oemetadata-upload => tutorial/data/oemetadata-upload}/series.json (100%) rename {oemetadata-upload => tutorial/data/oemetadata-upload}/timespans.json (100%) rename {oemetadata-upload => tutorial/data/oemetadata-upload}/variables.json (100%) diff --git a/datapackage_example/OEDataModel-normalization-datapackage.json b/tutorial/data/datapackage_example/OEDataModel-normalization-datapackage.json similarity index 100% rename from datapackage_example/OEDataModel-normalization-datapackage.json rename to tutorial/data/datapackage_example/OEDataModel-normalization-datapackage.json diff --git a/oemetadata-upload/flags.json b/tutorial/data/oemetadata-upload/flags.json similarity index 100% rename from oemetadata-upload/flags.json rename to tutorial/data/oemetadata-upload/flags.json diff --git a/oemetadata-upload/locations.json b/tutorial/data/oemetadata-upload/locations.json similarity index 100% rename from oemetadata-upload/locations.json rename to tutorial/data/oemetadata-upload/locations.json diff --git a/oemetadata-upload/series.json b/tutorial/data/oemetadata-upload/series.json similarity index 100% rename from oemetadata-upload/series.json rename to tutorial/data/oemetadata-upload/series.json diff --git a/oemetadata-upload/timespans.json b/tutorial/data/oemetadata-upload/timespans.json similarity index 100% rename from oemetadata-upload/timespans.json rename to tutorial/data/oemetadata-upload/timespans.json diff --git a/oemetadata-upload/variables.json b/tutorial/data/oemetadata-upload/variables.json similarity index 100% rename from oemetadata-upload/variables.json rename to tutorial/data/oemetadata-upload/variables.json From d22e087e51e60e5e18a5fc33aa41acef1c008410 Mon Sep 17 00:00:00 2001 From: Jonas Huber Date: Tue, 16 Apr 2024 18:28:44 +0200 Subject: [PATCH 14/16] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30b8388..26c4950 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,7 +36,7 @@ Template: ### Changed -- +- Reworked repo file structure & extend documentation [#](https://github.com/OpenEnergyPlatform/oem2orm/pull/) ### Removed From 2dca76c37b02861d0bdb36be4bf36f8b2ed093f1 Mon Sep 17 00:00:00 2001 From: Jonas Huber Date: Tue, 16 Apr 2024 18:38:59 +0200 Subject: [PATCH 15/16] update release changelog --- CHANGELOG.md | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21c07a6..1f76a6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,23 +32,30 @@ Template: ### Added -- Metadata upload in case of single table in OEM - - ### Changed -- Reworked repo file structure & extend documentation [#](https://github.com/OpenEnergyPlatform/oem2orm/pull/) +- ### Removed ______________________________________________________________________ -## [v0.3.3] - 2024-04-15 +## [v0.4.0] - 2024-04-16 + +### Added + +- Metadata upload in case of single table in OEM [#21](https://github.com/OpenEnergyPlatform/oem2orm/pull/21) + +### Changed + +- Reworked repo file structure & extend documentation [#49](https://github.com/OpenEnergyPlatform/oem2orm/pull/49) ### Fixed -- oedialect version to v0.1.1 -- sqlalchemy version to v1.3.16 + +- oedialect version to v0.1.1 [#47](https://github.com/OpenEnergyPlatform/oem2orm/pull/47) +- sqlalchemy version to v1.3.16 [#47](https://github.com/OpenEnergyPlatform/oem2orm/pull/47) ______________________________________________________________________ From f18778a0706fac4dbf78a10e10d86ba9cc7e359c Mon Sep 17 00:00:00 2001 From: Jonas Huber Date: Tue, 16 Apr 2024 18:40:06 +0200 Subject: [PATCH 16/16] bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7f86f61..6398766 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="oem2orm", - version="0.3.3", + version="0.4.0", author="henhuy, jh-RLI", author_email="Hendrik.Huyskens@rl-institut.de", description="SQLAlchemy module to generate ORM, read from data model (oedatamodel) in open-energy-metadata JSON format",