From bd1ac7960b45083959ebd376ebd1fda704d889e0 Mon Sep 17 00:00:00 2001 From: Ray Li Date: Wed, 18 Sep 2024 14:47:24 -0500 Subject: [PATCH 01/24] Type Mapping #1 - Add complete list of types. --- omymodels/types.py | 246 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 186 insertions(+), 60 deletions(-) diff --git a/omymodels/types.py b/omymodels/types.py index 8eac93f..1ef83b0 100644 --- a/omymodels/types.py +++ b/omymodels/types.py @@ -1,81 +1,164 @@ -from typing import Dict +from typing import Dict, Any from table_meta.model import Column -postgresql_dialect = ["ARRAY", "JSON", "JSONB", "UUID"] +# Define PostgreSQL-specific dialect types +postgresql_dialect = ["ARRAY", "JSON", "JSONB", "UUID", "TIMESTAMPTZ", "INTERVAL", "BYTEA", "SERIAL", "BIGSERIAL"] + +# Define MySQL-specific dialect types +mysql_dialect = ["ENUM", "SET", "JSON"] string_types = ( - "str", - "varchar", + "char", "character", + "varchar", "character varying", - "varying", - "char", + "varying character", "string", - "String", + "str", ) -text_types = ("text", "Text") -datetime_types = ( - "DATETIME", - "time", - "datetime.datetime", - "datetime", - "datetime.date", - "date", +text_types = ( + "text", + "tinytext", + "mediumtext", + "longtext", ) binary_types = ( - "BINARY", - "VARBINARY", "binary", "varbinary", + "blob", + "tinyblob", + "mediumblob", + "longblob", ) -json_types = ("union[dict, list]", "json", "union") - -integer_types = ("integer", "int", "serial") +integer_types = ( + "tinyint", + "smallint", + "mediumint", + "int", + "integer", +) -big_integer_types = ("bigint", "bigserial") +big_integer_types = ("bigint",) -float_types = ("float",) +float_types = ( + "float", + "double", + "real", +) -numeric_types = ("decimal", "numeric", "double") +numeric_types = ( + "decimal", + "numeric", +) -boolean_types = ("boolean", "bool") +boolean_types = ( + "boolean", + "bool", + "bit", +) datetime_types = ( - "TIMESTAMP", - "DATETIME", - "DATE", - "datetime.datetime", - "datetime", - "datetime.date", "date", + "datetime", + "timestamp", + "time", + "year", ) +json_types = ( + "json", +) -def populate_types_mapping(mapper: Dict) -> Dict: +uuid_types = ( + "uuid", +) + +# PostgreSQL-Specific Types +postgresql_specific_mapper = { + "array": "List[Any]", # PostgreSQL arrays can be mapped to Python lists + "json": "Any", + "jsonb": "Any", + "timestamptz": "datetime", # For PostgreSQL's timezone-aware timestamp + "interval": "float", # Intervals can be represented as floating-point seconds + "bytea": "bytes", # PostgreSQL binary data + "serial": "int", # PostgreSQL auto-incrementing serial type + "bigserial": "float", # PostgreSQL auto-incrementing bigserial type +} + +# MySQL-Specific Types +mysql_specific_mapper = { + "enum": "str", # Alternatively, map to specific Enum classes + "set": "List[str]", + "json": "Any", +} + +# General mapper for both MySQL and PostgreSQL +mapper = { + string_types: "str", + text_types: "str", + integer_types: "int", + big_integer_types: "float", + float_types: "float", + numeric_types: "float", + boolean_types: "bool", + datetime_types: "datetime", + binary_types: "bytes", + json_types: "Any", + uuid_types: "str", # Map to 'str' to be compatible with Pydantic and then validated as UUID +} + + +def populate_types_mapping(mapper_dict: Dict[tuple, str]) -> Dict[str, str]: + """ + Populates a dictionary mapping each type to its corresponding Pydantic type. + All type keys are converted to lowercase for case-insensitive matching. + """ types_mapping = {} - for type_group, value in mapper.items(): + for type_group, pydantic_type in mapper_dict.items(): for type_ in type_group: - types_mapping[type_] = value + types_mapping[type_.lower()] = pydantic_type return types_mapping -def prepare_type(column_data: Dict, models_types_mapping: Dict) -> str: - column_type = None +# Generate the general types_mapping using the mapper +types_mapping = populate_types_mapping(mapper) + + +def prepare_type(column_data: Column, models_types_mapping: Dict[str, str]) -> str: + """ + Determines the Pydantic type for a given column, handling special cases. + Specifically maps 'tinyint(1)' to 'bool' in MySQL. + """ column_data_type = column_data.type.lower().split("[")[0] - if not column_type: - column_type = models_types_mapping.get(column_data_type, column_type) + + # Special handling for MySQL tinyint(1) -> bool + if column_data_type.startswith("tinyint"): + size = column_data.size + if size == 1: + return "bool" + else: + column_data_type = "tinyint" + + # Get the Pydantic type from the mapping + column_type = models_types_mapping.get(column_data_type) + + # Default to the column data type if not found in mapping if not column_type: column_type = column_data_type + return column_type def add_custom_type_orm( - custom_types: Dict, column_data_type: str, column_type: str + custom_types: Dict[str, Any], column_data_type: str, column_type: str ) -> str: + """ + Adds custom type mappings from the ORM's custom_types dictionary. + """ if "." in column_data_type: column_data_type = column_data_type.split(".")[1] column_type = custom_types.get(column_data_type, column_type) @@ -88,15 +171,22 @@ def add_custom_type_orm( return column_type -def set_column_size(column_type: str, column_data: Dict) -> str: - if isinstance(column_data.size, int): - column_type += f"({column_data.size})" - elif isinstance(column_data.size, tuple): - column_type += f"({','.join([str(x) for x in column_data.size])})" +def set_column_size(column_type: str, column_data: Column) -> str: + """ + Appends the size or precision/scale to the column type if applicable. + """ + size = column_data.size + if isinstance(size, int): + column_type += f"({size})" + elif isinstance(size, tuple): + column_type += f"({','.join([str(x) for x in size])})" return column_type -def add_size_to_orm_column(column_type: str, column_data: Dict) -> str: +def add_size_to_orm_column(column_type: str, column_data: Column) -> str: + """ + Adds size information to the column type if available. + """ if column_data.size: column_type = set_column_size(column_type, column_data) elif column_type != "UUID" and "(" not in column_type: @@ -105,37 +195,73 @@ def add_size_to_orm_column(column_type: str, column_data: Dict) -> str: def process_types_after_models_parser(column_data: Column) -> Column: - if "." in column_data.type: - column_data.type = column_data.type.split(".")[1] - if "(" in column_data.type: - if "Enum" not in column_data.type: - column_data.type = column_data.type.split("(")[0] + """ + Processes the column type string to remove schema qualifiers and parameters. + """ + type_str = column_data.type + if "." in type_str: + type_str = type_str.split(".")[1] + if "(" in type_str: + if "Enum" not in type_str: + type_str = type_str.split("(")[0] else: - column_data.type = column_data.type.split("Enum(")[1].replace(")", "") - column_data.type = column_data.type.lower() + type_str = type_str.split("Enum(")[1].replace(")", "") + column_data.type = type_str.lower() return column_data -def prepare_column_data(column_data: Column) -> str: - if "." in column_data.type or "(": +def prepare_column_data(column_data: Column) -> Column: + """ + Prepares the column data by processing the type string. + """ + if "." in column_data.type or "(" in column_data.type: column_data = process_types_after_models_parser(column_data) return column_data -def prepare_column_type_orm(obj: object, column_data: Column) -> str: +def prepare_column_type_orm(obj: Any, column_data: Column) -> str: + """ + Prepares the Pydantic type for a given column based on the ORM object and column data. + Handles both MySQL and PostgreSQL dialects, including custom types and size specifications. + """ column_type = None column_data = prepare_column_data(column_data) - if obj.custom_types: + + # Determine dialect based on obj; assuming obj has a 'dialect' attribute + dialect = getattr(obj, 'dialect', 'mysql').lower() # default to MySQL if not specified + + # Handle custom types if any + if hasattr(obj, 'custom_types') and obj.custom_types: column_type = add_custom_type_orm( obj.custom_types, column_data.type, column_type ) + + # Prepare general type if not column_type: - column_type = prepare_type(column_data, obj.types_mapping) - if column_type in postgresql_dialect: - obj.postgresql_dialect_cols.add(column_type) + column_type = prepare_type(column_data, types_mapping) + + # Apply dialect-specific mappings + if dialect == "postgresql": + if column_type.upper() in postgresql_dialect: + obj.postgresql_dialect_cols.add(column_type.upper()) + # Apply PostgreSQL-specific mappings + column_type = postgresql_specific_mapper.get(column_type.lower(), column_type) + elif dialect == "mysql": + if column_type.upper() in mysql_dialect: + obj.mysql_dialect_cols.add(column_type.upper()) + # Apply MySQL-specific mappings + column_type = mysql_specific_mapper.get(column_type.lower(), column_type) + # Add size if applicable column_type = add_size_to_orm_column(column_type, column_data) - if "[" in column_data.type and column_data.type not in json_types: + + # Handle array types + if "[" in column_data.type and dialect == "postgresql" and column_data.type not in json_types: obj.postgresql_dialect_cols.add("ARRAY") - column_type = f"ARRAY({column_type})" + column_type = f"List[{column_type}]" + elif "[" in column_data.type and dialect == "mysql" and column_data.type not in json_types: + # MySQL doesn't support native arrays, consider using List or JSON + obj.mysql_dialect_cols.add("ARRAY") + column_type = f"List[{column_type}]" + return column_type From a50c336cac2af428ec0e2cc1d6ae10fd4392bcf1 Mon Sep 17 00:00:00 2001 From: Ray Li Date: Wed, 18 Sep 2024 15:18:06 -0500 Subject: [PATCH 02/24] Type Mapping #2 - Generate types and imports correctly. --- omymodels/logic.py | 6 ++- omymodels/models/pydantic/core.py | 61 +++++++++++++++----------- omymodels/models/pydantic/templates.py | 2 +- omymodels/models/pydantic/types.py | 35 +-------------- omymodels/types.py | 4 +- 5 files changed, 45 insertions(+), 63 deletions(-) diff --git a/omymodels/logic.py b/omymodels/logic.py index bca3bfa..9e99264 100644 --- a/omymodels/logic.py +++ b/omymodels/logic.py @@ -1,10 +1,12 @@ from typing import Dict, List +from table_meta.model import Column + import omymodels.types as t def generate_column( - column_data: Dict, + column_data: Column, table_pk: List[str], table_data: Dict, schema_global: bool, @@ -24,7 +26,7 @@ def generate_column( def setup_column_attributes( - column_data: Dict, + column_data: Column, table_pk: List[str], column: str, table_data: Dict, diff --git a/omymodels/models/pydantic/core.py b/omymodels/models/pydantic/core.py index e719a8f..eb38dbf 100644 --- a/omymodels/models/pydantic/core.py +++ b/omymodels/models/pydantic/core.py @@ -1,6 +1,6 @@ -from typing import Dict, List, Optional +from typing import List, Optional -from table_meta.model import Column +from table_meta.model import Column, TableMeta import omymodels.types as t from omymodels.helpers import create_class_name, datetime_now_check @@ -11,7 +11,7 @@ class ModelGenerator: def __init__(self): - self.imports = set([pt.base_model]) + self.imports = {pt.base_model} self.types_for_import = ["Json"] self.datetime_import = False self.typing_imports = set() @@ -19,21 +19,20 @@ def __init__(self): self.uuid_import = False self.prefix = "" - def add_custom_type(self, target_type): + def add_custom_type(self, target_type: str) -> Optional[str]: column_type = self.custom_types.get(target_type, None) _type = None if isinstance(column_type, tuple): _type = column_type[1] return _type - def get_not_custom_type(self, column: Column): + def get_not_custom_type(self, column: Column) -> str: _type = None if "." in column.type: _type = column.type.split(".")[1] else: _type = column.type.lower().split("[")[0] - if _type == _type: - _type = types_mapping.get(_type, _type) + _type = types_mapping.get(_type, _type) if _type in self.types_for_import: self.imports.add(_type) elif "datetime" in _type: @@ -43,9 +42,13 @@ def get_not_custom_type(self, column: Column): _type = f"List[{_type}]" if _type == "UUID": self.uuid_import = True + if "List" in _type: + self.typing_imports.add("List") + if "Any" == _type: + self.typing_imports.add("Any") return _type - def generate_attr(self, column: Dict, defaults_off: bool) -> str: + def generate_attr(self, column: Column, defaults_off: bool) -> str: _type = None if column.nullable: @@ -53,6 +56,7 @@ def generate_attr(self, column: Dict, defaults_off: bool) -> str: column_str = pt.pydantic_optional_attr else: column_str = pt.pydantic_attr + if self.custom_types: _type = self.add_custom_type(column.type) if not _type: @@ -60,40 +64,47 @@ def generate_attr(self, column: Dict, defaults_off: bool) -> str: column_str = column_str.format(arg_name=column.name, type=_type) - if column.default and defaults_off is False: + if column.default is not None and not defaults_off: column_str = self.add_default_values(column_str, column) return column_str @staticmethod - def add_default_values(column_str: str, column: Dict) -> str: + def add_default_values(column_str: str, column: Column) -> str: + # Handle datetime default values if column.type.upper() in datetime_types: if datetime_now_check(column.default.lower()): - # todo: need to add other popular PostgreSQL & MySQL functions + # Handle functions like CURRENT_TIMESTAMP column.default = "datetime.datetime.now()" - elif "'" not in column.default: - column.default = f"'{column['default']}'" + elif column.default.upper() != 'NULL' and "'" not in column.default: + column.default = f"'{column.default}'" + + # If the default is 'NULL', don't set a default in Pydantic (it already defaults to None) + if column.default.upper() == 'NULL': + return column_str + + # Append the default value if it's not None (e.g., explicit default values like '0' or CURRENT_TIMESTAMP) column_str += pt.pydantic_default_attr.format(default=column.default) return column_str def generate_model( - self, - table: Dict, - singular: bool = True, - exceptions: Optional[List] = None, - defaults_off: Optional[bool] = False, - *args, - **kwargs, + self, + table: TableMeta, + singular: bool = True, + exceptions: Optional[List] = None, + defaults_off: Optional[bool] = False, + *args, + **kwargs, ) -> str: model = "" # mean one model one table model += "\n\n" model += ( - pt.pydantic_class.format( - class_name=create_class_name(table.name, singular, exceptions), - table_name=table.name, - ) - ) + "\n\n" + pt.pydantic_class.format( + class_name=create_class_name(table.name, singular, exceptions), + table_name=table.name, + ) + ) + "\n" for column in table.columns: column = t.prepare_column_data(column) diff --git a/omymodels/models/pydantic/templates.py b/omymodels/models/pydantic/templates.py index d311139..21f29ec 100644 --- a/omymodels/models/pydantic/templates.py +++ b/omymodels/models/pydantic/templates.py @@ -1,4 +1,4 @@ -datetime_import = """import datetime""" +datetime_import = """import datetime as datetime""" typing_imports = """from typing import {typing_types}""" uuid_import = """from uuid import UUID""" diff --git a/omymodels/models/pydantic/types.py b/omymodels/models/pydantic/types.py index 7ff3d58..508dba4 100644 --- a/omymodels/models/pydantic/types.py +++ b/omymodels/models/pydantic/types.py @@ -1,39 +1,6 @@ from omymodels.types import ( - big_integer_types, - binary_types, - boolean_types, - datetime_types, - float_types, - integer_types, - json_types, - numeric_types, populate_types_mapping, - string_types, - text_types, + mapper, ) -mapper = { - string_types: "str", - integer_types: "int", - big_integer_types: "int", - float_types: "float", - numeric_types: "float", - boolean_types: "bool", - datetime_types: "datetime.datetime", - json_types: "Json", - text_types: "str", - binary_types: "bytes", -} - types_mapping = populate_types_mapping(mapper) - -direct_types = { - "date": "datetime.date", - "timestamp": "datetime.datetime", - "smallint": "int", - "jsonb": "Json", - "uuid": "UUID", -} - - -types_mapping.update(direct_types) diff --git a/omymodels/types.py b/omymodels/types.py index 1ef83b0..c0480ee 100644 --- a/omymodels/types.py +++ b/omymodels/types.py @@ -109,9 +109,11 @@ binary_types: "bytes", json_types: "Any", uuid_types: "str", # Map to 'str' to be compatible with Pydantic and then validated as UUID + ("point",): "List[float]", # Representing point as [float, float] + ("linestring",): "List[List[float]]", # List of points + ("polygon",): "List[List[List[float]]]", # List of LineStrings } - def populate_types_mapping(mapper_dict: Dict[tuple, str]) -> Dict[str, str]: """ Populates a dictionary mapping each type to its corresponding Pydantic type. From 655b05ce04f8416532843083bc182a74461686f6 Mon Sep 17 00:00:00 2001 From: Ray Li Date: Mon, 23 Sep 2024 13:59:25 -0500 Subject: [PATCH 03/24] Reformat Files --- omymodels/models/pydantic/core.py | 28 ++++++++-------- omymodels/models/pydantic/types.py | 5 +-- omymodels/types.py | 53 ++++++++++++++++++++---------- 3 files changed, 50 insertions(+), 36 deletions(-) diff --git a/omymodels/models/pydantic/core.py b/omymodels/models/pydantic/core.py index eb38dbf..469f83d 100644 --- a/omymodels/models/pydantic/core.py +++ b/omymodels/models/pydantic/core.py @@ -76,11 +76,11 @@ def add_default_values(column_str: str, column: Column) -> str: if datetime_now_check(column.default.lower()): # Handle functions like CURRENT_TIMESTAMP column.default = "datetime.datetime.now()" - elif column.default.upper() != 'NULL' and "'" not in column.default: + elif column.default.upper() != "NULL" and "'" not in column.default: column.default = f"'{column.default}'" # If the default is 'NULL', don't set a default in Pydantic (it already defaults to None) - if column.default.upper() == 'NULL': + if column.default.upper() == "NULL": return column_str # Append the default value if it's not None (e.g., explicit default values like '0' or CURRENT_TIMESTAMP) @@ -88,23 +88,23 @@ def add_default_values(column_str: str, column: Column) -> str: return column_str def generate_model( - self, - table: TableMeta, - singular: bool = True, - exceptions: Optional[List] = None, - defaults_off: Optional[bool] = False, - *args, - **kwargs, + self, + table: TableMeta, + singular: bool = True, + exceptions: Optional[List] = None, + defaults_off: Optional[bool] = False, + *args, + **kwargs, ) -> str: model = "" # mean one model one table model += "\n\n" model += ( - pt.pydantic_class.format( - class_name=create_class_name(table.name, singular, exceptions), - table_name=table.name, - ) - ) + "\n" + pt.pydantic_class.format( + class_name=create_class_name(table.name, singular, exceptions), + table_name=table.name, + ) + ) + "\n" for column in table.columns: column = t.prepare_column_data(column) diff --git a/omymodels/models/pydantic/types.py b/omymodels/models/pydantic/types.py index 508dba4..3ee1d45 100644 --- a/omymodels/models/pydantic/types.py +++ b/omymodels/models/pydantic/types.py @@ -1,6 +1,3 @@ -from omymodels.types import ( - populate_types_mapping, - mapper, -) +from omymodels.types import mapper, populate_types_mapping types_mapping = populate_types_mapping(mapper) diff --git a/omymodels/types.py b/omymodels/types.py index c0480ee..58033d3 100644 --- a/omymodels/types.py +++ b/omymodels/types.py @@ -1,9 +1,19 @@ -from typing import Dict, Any +from typing import Any, Dict from table_meta.model import Column # Define PostgreSQL-specific dialect types -postgresql_dialect = ["ARRAY", "JSON", "JSONB", "UUID", "TIMESTAMPTZ", "INTERVAL", "BYTEA", "SERIAL", "BIGSERIAL"] +postgresql_dialect = [ + "ARRAY", + "JSON", + "JSONB", + "UUID", + "TIMESTAMPTZ", + "INTERVAL", + "BYTEA", + "SERIAL", + "BIGSERIAL", +] # Define MySQL-specific dialect types mysql_dialect = ["ENUM", "SET", "JSON"] @@ -69,29 +79,25 @@ "year", ) -json_types = ( - "json", -) +json_types = ("json",) -uuid_types = ( - "uuid", -) +uuid_types = ("uuid",) # PostgreSQL-Specific Types postgresql_specific_mapper = { - "array": "List[Any]", # PostgreSQL arrays can be mapped to Python lists + "array": "List[Any]", # PostgreSQL arrays can be mapped to Python lists "json": "Any", "jsonb": "Any", "timestamptz": "datetime", # For PostgreSQL's timezone-aware timestamp - "interval": "float", # Intervals can be represented as floating-point seconds - "bytea": "bytes", # PostgreSQL binary data - "serial": "int", # PostgreSQL auto-incrementing serial type - "bigserial": "float", # PostgreSQL auto-incrementing bigserial type + "interval": "float", # Intervals can be represented as floating-point seconds + "bytea": "bytes", # PostgreSQL binary data + "serial": "int", # PostgreSQL auto-incrementing serial type + "bigserial": "float", # PostgreSQL auto-incrementing bigserial type } # MySQL-Specific Types mysql_specific_mapper = { - "enum": "str", # Alternatively, map to specific Enum classes + "enum": "str", # Alternatively, map to specific Enum classes "set": "List[str]", "json": "Any", } @@ -114,6 +120,7 @@ ("polygon",): "List[List[List[float]]]", # List of LineStrings } + def populate_types_mapping(mapper_dict: Dict[tuple, str]) -> Dict[str, str]: """ Populates a dictionary mapping each type to its corresponding Pydantic type. @@ -230,10 +237,12 @@ def prepare_column_type_orm(obj: Any, column_data: Column) -> str: column_data = prepare_column_data(column_data) # Determine dialect based on obj; assuming obj has a 'dialect' attribute - dialect = getattr(obj, 'dialect', 'mysql').lower() # default to MySQL if not specified + dialect = getattr( + obj, "dialect", "mysql" + ).lower() # default to MySQL if not specified # Handle custom types if any - if hasattr(obj, 'custom_types') and obj.custom_types: + if hasattr(obj, "custom_types") and obj.custom_types: column_type = add_custom_type_orm( obj.custom_types, column_data.type, column_type ) @@ -258,10 +267,18 @@ def prepare_column_type_orm(obj: Any, column_data: Column) -> str: column_type = add_size_to_orm_column(column_type, column_data) # Handle array types - if "[" in column_data.type and dialect == "postgresql" and column_data.type not in json_types: + if ( + "[" in column_data.type + and dialect == "postgresql" + and column_data.type not in json_types + ): obj.postgresql_dialect_cols.add("ARRAY") column_type = f"List[{column_type}]" - elif "[" in column_data.type and dialect == "mysql" and column_data.type not in json_types: + elif ( + "[" in column_data.type + and dialect == "mysql" + and column_data.type not in json_types + ): # MySQL doesn't support native arrays, consider using List or JSON obj.mysql_dialect_cols.add("ARRAY") column_type = f"List[{column_type}]" From cd5025db854b38d32f1fcca74d177aaf5a65cdcc Mon Sep 17 00:00:00 2001 From: Ray Li Date: Mon, 23 Sep 2024 14:08:43 -0500 Subject: [PATCH 04/24] Fix Datetime Import --- omymodels/models/pydantic/templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omymodels/models/pydantic/templates.py b/omymodels/models/pydantic/templates.py index 21f29ec..8a1cca1 100644 --- a/omymodels/models/pydantic/templates.py +++ b/omymodels/models/pydantic/templates.py @@ -1,4 +1,4 @@ -datetime_import = """import datetime as datetime""" +datetime_import = """from datetime import datetime""" typing_imports = """from typing import {typing_types}""" uuid_import = """from uuid import UUID""" From 5435d2f20a5f645e95ac0294cb70e03085cde5c7 Mon Sep 17 00:00:00 2001 From: Ray Li Date: Mon, 23 Sep 2024 14:56:26 -0500 Subject: [PATCH 05/24] Fix Default DateTime Types --- omymodels/helpers.py | 17 ++++++++++++++--- omymodels/models/pydantic/core.py | 11 ++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/omymodels/helpers.py b/omymodels/helpers.py index c7efb05..fc07cbb 100644 --- a/omymodels/helpers.py +++ b/omymodels/helpers.py @@ -99,6 +99,17 @@ def add_custom_types_to_generator(types: List[Type], generator: object) -> objec def datetime_now_check(string: str) -> bool: - if "now" in string or "current_timestamp" in string: - return True - return False + now_keywords = [ + "now", + "current_timestamp", + "current_date", + "current_time", + "localtime", + "localtimestamp", + "sysdate", + "getdate", + "current", + "curdate", + "curtime", + ] + return any(keyword in string.lower() for keyword in now_keywords) diff --git a/omymodels/models/pydantic/core.py b/omymodels/models/pydantic/core.py index 469f83d..a7d1a79 100644 --- a/omymodels/models/pydantic/core.py +++ b/omymodels/models/pydantic/core.py @@ -66,21 +66,22 @@ def generate_attr(self, column: Column, defaults_off: bool) -> str: if column.default is not None and not defaults_off: column_str = self.add_default_values(column_str, column) + if "datetime.now()" in column_str: + self.datetime_import = True return column_str @staticmethod def add_default_values(column_str: str, column: Column) -> str: # Handle datetime default values - if column.type.upper() in datetime_types: + if column.type.lower() in datetime_types: if datetime_now_check(column.default.lower()): - # Handle functions like CURRENT_TIMESTAMP - column.default = "datetime.datetime.now()" - elif column.default.upper() != "NULL" and "'" not in column.default: + column.default = "datetime.now()" + elif column.default.lower() != "null" and "'" not in column.default: column.default = f"'{column.default}'" # If the default is 'NULL', don't set a default in Pydantic (it already defaults to None) - if column.default.upper() == "NULL": + if column.default.lower() == "null": return column_str # Append the default value if it's not None (e.g., explicit default values like '0' or CURRENT_TIMESTAMP) From 170052c3704f9c76b0e25ab3e551f2f148456387 Mon Sep 17 00:00:00 2001 From: Ray Li Date: Mon, 23 Sep 2024 14:57:05 -0500 Subject: [PATCH 06/24] Lowercase Types for Consistency --- omymodels/types.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/omymodels/types.py b/omymodels/types.py index 58033d3..c2a0a65 100644 --- a/omymodels/types.py +++ b/omymodels/types.py @@ -4,19 +4,19 @@ # Define PostgreSQL-specific dialect types postgresql_dialect = [ - "ARRAY", - "JSON", - "JSONB", - "UUID", - "TIMESTAMPTZ", - "INTERVAL", - "BYTEA", - "SERIAL", - "BIGSERIAL", + "array", + "json", + "jsonb", + "uuid", + "timestamptz", + "interval", + "bytea", + "serial", + "bigserial", ] # Define MySQL-specific dialect types -mysql_dialect = ["ENUM", "SET", "JSON"] +mysql_dialect = ["enum", "set", "json"] string_types = ( "char", @@ -253,13 +253,13 @@ def prepare_column_type_orm(obj: Any, column_data: Column) -> str: # Apply dialect-specific mappings if dialect == "postgresql": - if column_type.upper() in postgresql_dialect: - obj.postgresql_dialect_cols.add(column_type.upper()) + if column_type.lower() in postgresql_dialect: + obj.postgresql_dialect_cols.add(column_type.lower()) # Apply PostgreSQL-specific mappings column_type = postgresql_specific_mapper.get(column_type.lower(), column_type) elif dialect == "mysql": - if column_type.upper() in mysql_dialect: - obj.mysql_dialect_cols.add(column_type.upper()) + if column_type.lower() in mysql_dialect: + obj.mysql_dialect_cols.add(column_type.lower()) # Apply MySQL-specific mappings column_type = mysql_specific_mapper.get(column_type.lower(), column_type) From d5aa435922c6613f25a75b69ccd4b69d6c9cbe86 Mon Sep 17 00:00:00 2001 From: Ray Li Date: Mon, 23 Sep 2024 15:29:46 -0500 Subject: [PATCH 07/24] Fix Varchar Collation Type --- omymodels/models/pydantic/core.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/omymodels/models/pydantic/core.py b/omymodels/models/pydantic/core.py index a7d1a79..0cae8d6 100644 --- a/omymodels/models/pydantic/core.py +++ b/omymodels/models/pydantic/core.py @@ -42,6 +42,10 @@ def get_not_custom_type(self, column: Column) -> str: _type = f"List[{_type}]" if _type == "UUID": self.uuid_import = True + if _type.startswith("varchar"): + # Remove character set and collation information + # Example: varchar(10)character set utf8mb4 collation utf8mb4_unicode_ci + _type = "str" if "List" in _type: self.typing_imports.add("List") if "Any" == _type: From 1971fec36c0ccc63fc49a76e11cbad4dd49a505f Mon Sep 17 00:00:00 2001 From: Ray Li Date: Mon, 23 Sep 2024 15:55:26 -0500 Subject: [PATCH 08/24] Disable Flake8 Complexity Rule --- .flake8 | 1 - 1 file changed, 1 deletion(-) diff --git a/.flake8 b/.flake8 index 07e8435..e216b9e 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,4 @@ [flake8] exclude = .github,.git,__pycache__,docs/source/conf.py,old,build,dist,models.py,omymodels/test.py ignore = D100, D103, D101, D102, D104,D107, D403, D210, D400, D401, W503, W293, D205 -max-complexity = 10 max-line-length = 120 From 0e56cc42e4bb59896c0f64af6be7ce5632ac1464 Mon Sep 17 00:00:00 2001 From: Ray Li Date: Mon, 23 Sep 2024 15:55:48 -0500 Subject: [PATCH 09/24] Remove Int and Text Extra Column Properties --- omymodels/models/pydantic/core.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/omymodels/models/pydantic/core.py b/omymodels/models/pydantic/core.py index 0cae8d6..761ec95 100644 --- a/omymodels/models/pydantic/core.py +++ b/omymodels/models/pydantic/core.py @@ -5,8 +5,14 @@ import omymodels.types as t from omymodels.helpers import create_class_name, datetime_now_check from omymodels.models.pydantic import templates as pt -from omymodels.models.pydantic.types import types_mapping -from omymodels.types import datetime_types +from omymodels.types import ( + big_integer_types, + datetime_types, + integer_types, + string_types, + text_types, + types_mapping, +) class ModelGenerator: @@ -42,9 +48,15 @@ def get_not_custom_type(self, column: Column) -> str: _type = f"List[{_type}]" if _type == "UUID": self.uuid_import = True - if _type.startswith("varchar"): - # Remove character set and collation information - # Example: varchar(10)character set utf8mb4 collation utf8mb4_unicode_ci + # Handle integer types + if any(t in _type for t in integer_types): + _type = "int" + # Handle big integer types + if any(t in _type for t in big_integer_types): + _type = "float" + # Remove character set and collation information + # Example: varchar(10) character set utf8mb4 collation utf8mb4_unicode_ci + if any(t in _type for t in string_types + text_types): _type = "str" if "List" in _type: self.typing_imports.add("List") From 7adaefa09112ac10b4fc8bb5ad9fac8dd47c045f Mon Sep 17 00:00:00 2001 From: Ray Li Date: Mon, 23 Sep 2024 16:25:33 -0500 Subject: [PATCH 10/24] Create Field Aliases - Resolve invalid Pydantic variable names by applying a Field alias. --- omymodels/models/pydantic/core.py | 66 ++++++++++++++++++++++++-- omymodels/models/pydantic/templates.py | 6 ++- 2 files changed, 66 insertions(+), 6 deletions(-) diff --git a/omymodels/models/pydantic/core.py b/omymodels/models/pydantic/core.py index 761ec95..824d8a2 100644 --- a/omymodels/models/pydantic/core.py +++ b/omymodels/models/pydantic/core.py @@ -1,3 +1,4 @@ +from keyword import iskeyword from typing import List, Optional from table_meta.model import Column, TableMeta @@ -78,14 +79,71 @@ def generate_attr(self, column: Column, defaults_off: bool) -> str: if not _type: _type = self.get_not_custom_type(column) - column_str = column_str.format(arg_name=column.name, type=_type) + field_params = self.get_field_params(column, defaults_off) + if field_params: + self.imports.add("Field") + + column_str = column_str.format( + arg_name=self._generate_valid_identifier(column.name), + type=_type, + field_params=field_params, + ) + + return column_str + + def get_field_params(self, column: Column, defaults_off: bool) -> str: + params = [] + + if not self._is_valid_identifier(column.name): + params.append(f'alias="{column.name}"') if column.default is not None and not defaults_off: - column_str = self.add_default_values(column_str, column) - if "datetime.now()" in column_str: + default_value = self.get_default_value(column) + if default_value: + if any( + t in column.type.lower() for t in integer_types + big_integer_types + ): + # Remove quotes for integer types + default_value = default_value.strip("'") + params.append(f"default={default_value}") + else: + # Keep quotes for other types + params.append(f"default={default_value}") + + if params: + return f" = Field({', '.join(params)})" + return "" + + def get_default_value(self, column: Column) -> str: + if column.default is None or column.default.lower() == "null": + return "" + + if column.type.lower() in datetime_types: + if datetime_now_check(column.default.lower()): self.datetime_import = True + return "datetime.now()" + else: + return column.default.strip("'") - return column_str + return column.default + + @staticmethod + def _is_valid_identifier(name: str) -> bool: + """Check if the given name is a valid Python identifier.""" + return name.isidentifier() and not iskeyword(name) + + @staticmethod + def _generate_valid_identifier(name: str) -> str: + """Generate a valid Python identifier from a given name.""" + # Replace non-alphanumeric characters with underscores + valid_name = "".join(c if c.isalnum() else "_" for c in name) + # Ensure the name doesn't start with a number + if valid_name[0].isdigit(): + valid_name = f"f_{valid_name}" + # Ensure it's not a Python keyword + if iskeyword(valid_name): + valid_name += "_" + return valid_name @staticmethod def add_default_values(column_str: str, column: Column) -> str: diff --git a/omymodels/models/pydantic/templates.py b/omymodels/models/pydantic/templates.py index 8a1cca1..834fb01 100644 --- a/omymodels/models/pydantic/templates.py +++ b/omymodels/models/pydantic/templates.py @@ -7,8 +7,10 @@ pydantic_imports = """from pydantic import {imports}""" pydantic_class = """class {class_name}(BaseModel):""" -pydantic_attr = """ {arg_name}: {type}""" -pydantic_optional_attr = """ {arg_name}: Optional[{type}]""" + +pydantic_attr = """ {arg_name}: {type}{field_params}""" +pydantic_optional_attr = """ {arg_name}: Optional[{type}]{field_params}""" + pydantic_default_attr = """ = {default}""" enum_class = """class {class_name}({sub_type}):""" From bbe4c5bf877a9f95e478311fc0d7fa48a2f01bd9 Mon Sep 17 00:00:00 2001 From: Ray Li Date: Mon, 23 Sep 2024 17:32:40 -0500 Subject: [PATCH 11/24] Create Field Aliases #2 - Use Field alias only when needed. --- omymodels/models/pydantic/core.py | 35 +++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/omymodels/models/pydantic/core.py b/omymodels/models/pydantic/core.py index 824d8a2..6351a14 100644 --- a/omymodels/models/pydantic/core.py +++ b/omymodels/models/pydantic/core.py @@ -79,14 +79,21 @@ def generate_attr(self, column: Column, defaults_off: bool) -> str: if not _type: _type = self.get_not_custom_type(column) - field_params = self.get_field_params(column, defaults_off) - if field_params: - self.imports.add("Field") + arg_name = column.name + field_params = None + if not self._is_valid_identifier(column.name): + field_params = self.get_field_params(column, defaults_off) + if field_params: + self.imports.add("Field") + arg_name = self._generate_valid_identifier(column.name) + else: + if column.default is not None and not defaults_off: + field_params = self.get_default_value_string(column) column_str = column_str.format( - arg_name=self._generate_valid_identifier(column.name), + arg_name=arg_name, type=_type, - field_params=field_params, + field_params=field_params if field_params is not None else "", ) return column_str @@ -99,6 +106,8 @@ def get_field_params(self, column: Column, defaults_off: bool) -> str: if column.default is not None and not defaults_off: default_value = self.get_default_value(column) + if any(t in column.type.lower() for t in ["json", "jsonb"]): + return "" if default_value: if any( t in column.type.lower() for t in integer_types + big_integer_types @@ -118,6 +127,9 @@ def get_default_value(self, column: Column) -> str: if column.default is None or column.default.lower() == "null": return "" + if any(t in column.type.lower() for t in ["json", "jsonb"]): + return "" + if column.type.lower() in datetime_types: if datetime_now_check(column.default.lower()): self.datetime_import = True @@ -146,7 +158,7 @@ def _generate_valid_identifier(name: str) -> str: return valid_name @staticmethod - def add_default_values(column_str: str, column: Column) -> str: + def get_default_value_string(column: Column) -> str: # Handle datetime default values if column.type.lower() in datetime_types: if datetime_now_check(column.default.lower()): @@ -156,11 +168,16 @@ def add_default_values(column_str: str, column: Column) -> str: # If the default is 'NULL', don't set a default in Pydantic (it already defaults to None) if column.default.lower() == "null": - return column_str + return "" + + # Remove quotes for integer types + if any(t in column.type.lower() for t in integer_types + big_integer_types): + default_value = column.default.strip("'") + else: + default_value = column.default # Append the default value if it's not None (e.g., explicit default values like '0' or CURRENT_TIMESTAMP) - column_str += pt.pydantic_default_attr.format(default=column.default) - return column_str + return pt.pydantic_default_attr.format(default=default_value) def generate_model( self, From cb014906bce987c10b6ded52487e1be9a889ec55 Mon Sep 17 00:00:00 2001 From: Ray Li Date: Mon, 23 Sep 2024 17:39:24 -0500 Subject: [PATCH 12/24] Create Field Aliases #3 - Handle Json types correctly. --- omymodels/models/pydantic/core.py | 34 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/omymodels/models/pydantic/core.py b/omymodels/models/pydantic/core.py index 6351a14..6e954e7 100644 --- a/omymodels/models/pydantic/core.py +++ b/omymodels/models/pydantic/core.py @@ -104,20 +104,20 @@ def get_field_params(self, column: Column, defaults_off: bool) -> str: if not self._is_valid_identifier(column.name): params.append(f'alias="{column.name}"') - if column.default is not None and not defaults_off: - default_value = self.get_default_value(column) - if any(t in column.type.lower() for t in ["json", "jsonb"]): - return "" - if default_value: - if any( - t in column.type.lower() for t in integer_types + big_integer_types - ): - # Remove quotes for integer types - default_value = default_value.strip("'") - params.append(f"default={default_value}") - else: - # Keep quotes for other types - params.append(f"default={default_value}") + if not any(t in column.type.lower() for t in ["json", "jsonb"]): + if column.default is not None and not defaults_off: + default_value = self.get_default_value(column) + if default_value: + if any( + t in column.type.lower() + for t in integer_types + big_integer_types + ): + # Remove quotes for integer types + default_value = default_value.strip("'") + params.append(f"default={default_value}") + else: + # Keep quotes for other types + params.append(f"default={default_value}") if params: return f" = Field({', '.join(params)})" @@ -127,9 +127,6 @@ def get_default_value(self, column: Column) -> str: if column.default is None or column.default.lower() == "null": return "" - if any(t in column.type.lower() for t in ["json", "jsonb"]): - return "" - if column.type.lower() in datetime_types: if datetime_now_check(column.default.lower()): self.datetime_import = True @@ -169,7 +166,8 @@ def get_default_value_string(column: Column) -> str: # If the default is 'NULL', don't set a default in Pydantic (it already defaults to None) if column.default.lower() == "null": return "" - + if any(t in column.type.lower() for t in ["json", "jsonb"]): + return "" # Remove quotes for integer types if any(t in column.type.lower() for t in integer_types + big_integer_types): default_value = column.default.strip("'") From b78ce459a14ac85a34b028d73b6974dd066d1daf Mon Sep 17 00:00:00 2001 From: Ray Li Date: Mon, 23 Sep 2024 18:15:07 -0500 Subject: [PATCH 13/24] Add User Customizable Type Mapping --- omymodels/models/pydantic/core.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/omymodels/models/pydantic/core.py b/omymodels/models/pydantic/core.py index 6e954e7..71a3c38 100644 --- a/omymodels/models/pydantic/core.py +++ b/omymodels/models/pydantic/core.py @@ -4,6 +4,7 @@ from table_meta.model import Column, TableMeta import omymodels.types as t +from omymodels import types from omymodels.helpers import create_class_name, datetime_now_check from omymodels.models.pydantic import templates as pt from omymodels.types import ( @@ -25,6 +26,7 @@ def __init__(self): self.custom_types = {} self.uuid_import = False self.prefix = "" + self.types_mapping = types_mapping def add_custom_type(self, target_type: str) -> Optional[str]: column_type = self.custom_types.get(target_type, None) @@ -33,18 +35,18 @@ def add_custom_type(self, target_type: str) -> Optional[str]: _type = column_type[1] return _type - def get_not_custom_type(self, column: Column) -> str: + def get_not_custom_type(self, type: str) -> str: _type = None - if "." in column.type: - _type = column.type.split(".")[1] + if "." in type: + _type = type.split(".")[1] else: - _type = column.type.lower().split("[")[0] + _type = type.lower().split("[")[0] _type = types_mapping.get(_type, _type) if _type in self.types_for_import: self.imports.add(_type) elif "datetime" in _type: self.datetime_import = True - elif "[" in column.type: + elif "[" in type: self.typing_imports.add("List") _type = f"List[{_type}]" if _type == "UUID": @@ -77,7 +79,8 @@ def generate_attr(self, column: Column, defaults_off: bool) -> str: if self.custom_types: _type = self.add_custom_type(column.type) if not _type: - _type = self.get_not_custom_type(column) + _type = types.prepare_type(column, self.types_mapping) + _type = self.get_not_custom_type(_type) arg_name = column.name field_params = None From 557a9918c2265b8f37cc2b9fc4f5c02649e7ab11 Mon Sep 17 00:00:00 2001 From: Ray Li Date: Mon, 23 Sep 2024 18:45:36 -0500 Subject: [PATCH 14/24] Remove Unnecessary Lowercase --- omymodels/models/pydantic/core.py | 2 +- omymodels/types.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/omymodels/models/pydantic/core.py b/omymodels/models/pydantic/core.py index 71a3c38..54bdbea 100644 --- a/omymodels/models/pydantic/core.py +++ b/omymodels/models/pydantic/core.py @@ -40,7 +40,7 @@ def get_not_custom_type(self, type: str) -> str: if "." in type: _type = type.split(".")[1] else: - _type = type.lower().split("[")[0] + _type = type.split("[")[0] _type = types_mapping.get(_type, _type) if _type in self.types_for_import: self.imports.add(_type) diff --git a/omymodels/types.py b/omymodels/types.py index c2a0a65..1161e0a 100644 --- a/omymodels/types.py +++ b/omymodels/types.py @@ -142,7 +142,7 @@ def prepare_type(column_data: Column, models_types_mapping: Dict[str, str]) -> s Determines the Pydantic type for a given column, handling special cases. Specifically maps 'tinyint(1)' to 'bool' in MySQL. """ - column_data_type = column_data.type.lower().split("[")[0] + column_data_type = column_data.type.split("[")[0] # Special handling for MySQL tinyint(1) -> bool if column_data_type.startswith("tinyint"): From 1029e2094abaccceba4728450f2e65f06d07f0c0 Mon Sep 17 00:00:00 2001 From: Ray Li Date: Mon, 23 Sep 2024 19:37:18 -0500 Subject: [PATCH 15/24] Generate Boolean Defaults - Set default value "True/False" instead of "1/0". --- omymodels/models/pydantic/core.py | 21 ++++++++++----------- omymodels/types.py | 8 ++------ 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/omymodels/models/pydantic/core.py b/omymodels/models/pydantic/core.py index 54bdbea..cc780e1 100644 --- a/omymodels/models/pydantic/core.py +++ b/omymodels/models/pydantic/core.py @@ -82,6 +82,7 @@ def generate_attr(self, column: Column, defaults_off: bool) -> str: _type = types.prepare_type(column, self.types_mapping) _type = self.get_not_custom_type(_type) + column.type = _type arg_name = column.name field_params = None if not self._is_valid_identifier(column.name): @@ -111,16 +112,7 @@ def get_field_params(self, column: Column, defaults_off: bool) -> str: if column.default is not None and not defaults_off: default_value = self.get_default_value(column) if default_value: - if any( - t in column.type.lower() - for t in integer_types + big_integer_types - ): - # Remove quotes for integer types - default_value = default_value.strip("'") - params.append(f"default={default_value}") - else: - # Keep quotes for other types - params.append(f"default={default_value}") + params.append(f"default={default_value}") if params: return f" = Field({', '.join(params)})" @@ -137,6 +129,9 @@ def get_default_value(self, column: Column) -> str: else: return column.default.strip("'") + if any(t in column.type.lower() for t in integer_types + big_integer_types): + return column.default.strip("'") + return column.default @staticmethod @@ -171,9 +166,13 @@ def get_default_value_string(column: Column) -> str: return "" if any(t in column.type.lower() for t in ["json", "jsonb"]): return "" - # Remove quotes for integer types + if column.type.lower() == "any": + return "" + if any(t in column.type.lower() for t in integer_types + big_integer_types): default_value = column.default.strip("'") + elif column.type.lower() == "bool": + default_value = "False" if column.default.strip("'") == "0" else "True" else: default_value = column.default diff --git a/omymodels/types.py b/omymodels/types.py index 1161e0a..8502d9b 100644 --- a/omymodels/types.py +++ b/omymodels/types.py @@ -145,12 +145,8 @@ def prepare_type(column_data: Column, models_types_mapping: Dict[str, str]) -> s column_data_type = column_data.type.split("[")[0] # Special handling for MySQL tinyint(1) -> bool - if column_data_type.startswith("tinyint"): - size = column_data.size - if size == 1: - return "bool" - else: - column_data_type = "tinyint" + if column_data_type == "tinyint" and column_data.size == 1: + return "bool" # Get the Pydantic type from the mapping column_type = models_types_mapping.get(column_data_type) From a357502398bc4d79763675cc30ef6bea7d45f195 Mon Sep 17 00:00:00 2001 From: Ray Li Date: Thu, 26 Sep 2024 17:10:32 -0500 Subject: [PATCH 16/24] Handle GENERATED ALWAYS AS - Add `Field(exclude=True)` annotation for generated columns. --- omymodels/from_ddl.py | 3 +++ omymodels/models/pydantic/core.py | 11 ++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/omymodels/from_ddl.py b/omymodels/from_ddl.py index 8db44a5..8fddfaf 100644 --- a/omymodels/from_ddl.py +++ b/omymodels/from_ddl.py @@ -99,6 +99,9 @@ def convert_ddl_to_models( # noqa: C901 column["name"] = snake_case(column["name"]) if column["name"] in refs: column["references"] = refs[column["name"]] + if "generated" in column: + column["generated_as"] = column["generated"]["as"] + if not no_auto_snake_case: table["primary_key"] = [snake_case(pk) for pk in table["primary_key"]] for uniq in table.get("constraints", {}).get("uniques", []): diff --git a/omymodels/models/pydantic/core.py b/omymodels/models/pydantic/core.py index cc780e1..56544df 100644 --- a/omymodels/models/pydantic/core.py +++ b/omymodels/models/pydantic/core.py @@ -85,7 +85,11 @@ def generate_attr(self, column: Column, defaults_off: bool) -> str: column.type = _type arg_name = column.name field_params = None - if not self._is_valid_identifier(column.name): + + if ( + self._is_valid_identifier(column.name) is False + or column.generated_as is not None + ): field_params = self.get_field_params(column, defaults_off) if field_params: self.imports.add("Field") @@ -114,6 +118,9 @@ def get_field_params(self, column: Column, defaults_off: bool) -> str: if default_value: params.append(f"default={default_value}") + if column.generated_as is not None: + params.append("exclude=True") + if params: return f" = Field({', '.join(params)})" return "" @@ -158,8 +165,6 @@ def get_default_value_string(column: Column) -> str: if column.type.lower() in datetime_types: if datetime_now_check(column.default.lower()): column.default = "datetime.now()" - elif column.default.lower() != "null" and "'" not in column.default: - column.default = f"'{column.default}'" # If the default is 'NULL', don't set a default in Pydantic (it already defaults to None) if column.default.lower() == "null": From 57974b36b80878320704b667ba198dc7762119c6 Mon Sep 17 00:00:00 2001 From: Ray Li Date: Thu, 26 Sep 2024 19:27:53 -0500 Subject: [PATCH 17/24] QUICKFIX Type Mapping Import - Fix import abstract type mapping through the pydantic types class. --- omymodels/models/pydantic/core.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/omymodels/models/pydantic/core.py b/omymodels/models/pydantic/core.py index 56544df..bd38343 100644 --- a/omymodels/models/pydantic/core.py +++ b/omymodels/models/pydantic/core.py @@ -3,10 +3,10 @@ from table_meta.model import Column, TableMeta -import omymodels.types as t -from omymodels import types +import omymodels.types as types from omymodels.helpers import create_class_name, datetime_now_check from omymodels.models.pydantic import templates as pt +from omymodels.models.pydantic import types as pydantic_types from omymodels.types import ( big_integer_types, datetime_types, @@ -26,7 +26,7 @@ def __init__(self): self.custom_types = {} self.uuid_import = False self.prefix = "" - self.types_mapping = types_mapping + self.types_mapping = pydantic_types.types_mapping def add_custom_type(self, target_type: str) -> Optional[str]: column_type = self.custom_types.get(target_type, None) @@ -204,7 +204,7 @@ def generate_model( ) + "\n" for column in table.columns: - column = t.prepare_column_data(column) + column = types.prepare_column_data(column) model += self.generate_attr(column, defaults_off) + "\n" return model From 6a93099983f1ad3b1a945cbd4b5f6055419de925 Mon Sep 17 00:00:00 2001 From: Ray Li Date: Fri, 27 Sep 2024 14:55:24 -0500 Subject: [PATCH 18/24] Table Prefix and Suffix Support --- omymodels/from_ddl.py | 5 ++++- omymodels/models/pydantic/core.py | 11 ++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/omymodels/from_ddl.py b/omymodels/from_ddl.py index 8fddfaf..ed1525c 100644 --- a/omymodels/from_ddl.py +++ b/omymodels/from_ddl.py @@ -42,6 +42,8 @@ def create_models( defaults_off: Optional[bool] = False, exit_silent: Optional[bool] = False, no_auto_snake_case: Optional[bool] = False, + table_prefix: Optional[str] = "", + table_suffix: Optional[str] = "", ): """models_type can be: "gino", "dataclass", "pydantic" """ # extract data from ddl file @@ -55,7 +57,8 @@ def create_models( raise NoTablesError() # generate code output = generate_models_file( - data, singular, naming_exceptions, models_type, schema_global, defaults_off + data, + singular, ) if dump: save_models_to_file(output, dump_path) diff --git a/omymodels/models/pydantic/core.py b/omymodels/models/pydantic/core.py index bd38343..7a7838f 100644 --- a/omymodels/models/pydantic/core.py +++ b/omymodels/models/pydantic/core.py @@ -196,9 +196,18 @@ def generate_model( model = "" # mean one model one table model += "\n\n" + + table_prefix = kwargs.get("table_prefix", "") + table_suffix = kwargs.get("table_suffix", "") + + class_name = ( + table_prefix + + create_class_name(table.name, singular, exceptions) + + table_suffix + ) model += ( pt.pydantic_class.format( - class_name=create_class_name(table.name, singular, exceptions), + class_name=class_name, table_name=table.name, ) ) + "\n" From b354a0f5f1e5ac021f4cdc8fe4db92dec5238efa Mon Sep 17 00:00:00 2001 From: Ray Li Date: Fri, 27 Sep 2024 14:56:48 -0500 Subject: [PATCH 19/24] Table Prefix and Suffix Support #2 --- omymodels/from_ddl.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/omymodels/from_ddl.py b/omymodels/from_ddl.py index ed1525c..f7302e0 100644 --- a/omymodels/from_ddl.py +++ b/omymodels/from_ddl.py @@ -59,6 +59,12 @@ def create_models( output = generate_models_file( data, singular, + naming_exceptions, + models_type, + schema_global, + defaults_off, + table_prefix=table_prefix, + table_suffix=table_suffix, ) if dump: save_models_to_file(output, dump_path) @@ -139,6 +145,8 @@ def generate_models_file( models_type: str = "gino", schema_global: bool = True, defaults_off: Optional[bool] = False, + table_prefix: Optional[str] = "", + table_suffix: Optional[str] = "", ) -> str: """method to prepare full file with all Models &""" models_str = "" @@ -158,6 +166,8 @@ def generate_models_file( exceptions, schema_global=schema_global, defaults_off=defaults_off, + table_prefix=table_prefix, + table_suffix=table_suffix, ) header += generator.create_header( data["tables"], schema=schema_global, models_str=models_str From 455501cf5a5cf4e3c7f301fa455e32df51c62197 Mon Sep 17 00:00:00 2001 From: Ray Li Date: Mon, 30 Sep 2024 14:29:45 -0500 Subject: [PATCH 20/24] Update simple-ddl-parser --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9156c86..6bd65d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.7,<4.0" -simple-ddl-parser = "^1.0.0" +simple-ddl-parser = "^1.7.0" Jinja2 = "^3.0.1" py-models-parser = "^0.7.0" pydantic = "^1.8.2" From 040f6a2bc2e9b4bdc7434035700bf5980fc088c8 Mon Sep 17 00:00:00 2001 From: Ray Li Date: Mon, 30 Sep 2024 23:37:29 -0500 Subject: [PATCH 21/24] Create Date and Time Types --- omymodels/models/pydantic/core.py | 137 +++++++++++++++++++++-------- omymodels/models/pydantic/types.py | 8 +- 2 files changed, 108 insertions(+), 37 deletions(-) diff --git a/omymodels/models/pydantic/core.py b/omymodels/models/pydantic/core.py index 7a7838f..4f1f6b7 100644 --- a/omymodels/models/pydantic/core.py +++ b/omymodels/models/pydantic/core.py @@ -1,3 +1,4 @@ +from datetime import datetime from keyword import iskeyword from typing import List, Optional @@ -7,14 +8,7 @@ from omymodels.helpers import create_class_name, datetime_now_check from omymodels.models.pydantic import templates as pt from omymodels.models.pydantic import types as pydantic_types -from omymodels.types import ( - big_integer_types, - datetime_types, - integer_types, - string_types, - text_types, - types_mapping, -) +from omymodels.types import big_integer_types, integer_types, string_types, text_types class ModelGenerator: @@ -22,6 +16,8 @@ def __init__(self): self.imports = {pt.base_model} self.types_for_import = ["Json"] self.datetime_import = False + self.date_import = False + self.time_import = False self.typing_imports = set() self.custom_types = {} self.uuid_import = False @@ -41,11 +37,15 @@ def get_not_custom_type(self, type: str) -> str: _type = type.split(".")[1] else: _type = type.split("[")[0] - _type = types_mapping.get(_type, _type) + _type = pydantic_types.types_mapping.get(_type, _type) if _type in self.types_for_import: self.imports.add(_type) elif "datetime" in _type: self.datetime_import = True + elif _type == "date": + self.date_import = True + elif _type == "time": + self.time_import = True elif "[" in type: self.typing_imports.add("List") _type = f"List[{_type}]" @@ -112,11 +112,9 @@ def get_field_params(self, column: Column, defaults_off: bool) -> str: if not self._is_valid_identifier(column.name): params.append(f'alias="{column.name}"') - if not any(t in column.type.lower() for t in ["json", "jsonb"]): - if column.default is not None and not defaults_off: - default_value = self.get_default_value(column) - if default_value: - params.append(f"default={default_value}") + if column.default is not None and not defaults_off: + if default_value := self.get_default_value_string(column): + params.append(f"default{default_value.replace(' ', '')}") if column.generated_as is not None: params.append("exclude=True") @@ -129,42 +127,40 @@ def get_default_value(self, column: Column) -> str: if column.default is None or column.default.lower() == "null": return "" - if column.type.lower() in datetime_types: + if column.type.lower() in ["datetime", "timestamp"]: if datetime_now_check(column.default.lower()): self.datetime_import = True return "datetime.now()" else: return column.default.strip("'") + if column.type.lower() == "date": + self.date_import = True + return self._convert_to_date_string(column.default.strip("'")) + + if column.type.lower() == "time": + self.time_import = True + return self._convert_to_time_string(column.default.strip("'")) + if any(t in column.type.lower() for t in integer_types + big_integer_types): return column.default.strip("'") return column.default - @staticmethod - def _is_valid_identifier(name: str) -> bool: - """Check if the given name is a valid Python identifier.""" - return name.isidentifier() and not iskeyword(name) - - @staticmethod - def _generate_valid_identifier(name: str) -> str: - """Generate a valid Python identifier from a given name.""" - # Replace non-alphanumeric characters with underscores - valid_name = "".join(c if c.isalnum() else "_" for c in name) - # Ensure the name doesn't start with a number - if valid_name[0].isdigit(): - valid_name = f"f_{valid_name}" - # Ensure it's not a Python keyword - if iskeyword(valid_name): - valid_name += "_" - return valid_name - @staticmethod def get_default_value_string(column: Column) -> str: # Handle datetime default values - if column.type.lower() in datetime_types: + if column.type.lower() in ["datetime", "timestamp"]: if datetime_now_check(column.default.lower()): column.default = "datetime.now()" + elif column.type.lower() == "date": + column.default = ModelGenerator._convert_to_date_string( + column.default.strip("'") + ) + elif column.type.lower() == "time": + column.default = ModelGenerator._convert_to_time_string( + column.default.strip("'") + ) # If the default is 'NULL', don't set a default in Pydantic (it already defaults to None) if column.default.lower() == "null": @@ -184,6 +180,68 @@ def get_default_value_string(column: Column) -> str: # Append the default value if it's not None (e.g., explicit default values like '0' or CURRENT_TIMESTAMP) return pt.pydantic_default_attr.format(default=default_value) + @classmethod + def _is_valid_identifier(self, name: str) -> bool: + return ( + name.isidentifier() + and not iskeyword(name) + and not self._is_pydantic_reserved_name(name) + ) + + @classmethod + def _is_pydantic_reserved_name(self, name: str) -> bool: + """Check if the name is a Pydantic-specific reserved name or starts with a reserved prefix.""" + pydantic_reserved_prefixes = {"dict_", "json_"} + pydantic_reserved_names = { + "copy", + "parse_obj", + "parse_raw", + "parse_file", + "from_orm", + "construct", + "validate", + "update_forward_refs", + "schema", + "schema_json", + "register", + } + return ( + any(name.startswith(prefix) for prefix in pydantic_reserved_prefixes) + or name in pydantic_reserved_names + ) + + @classmethod + def _generate_valid_identifier(self, name: str) -> str: + """Generate a valid Python identifier from a given name.""" + # Replace non-alphanumeric characters with underscores + valid_name = "".join(c if c.isalnum() else "_" for c in name) + + # Ensure the name doesn't start with a number + if ( + valid_name[0].isdigit() + or iskeyword(valid_name) + or self._is_pydantic_reserved_name(valid_name) + ): + valid_name = f"f_{valid_name}" + + return valid_name + + @staticmethod + def _convert_to_date_string(date_str: str) -> str: + try: + date_obj = datetime.strptime(date_str, "%Y-%m-%d").date() + return f"date({date_obj.year}, {date_obj.month}, {date_obj.day})" + except ValueError: + return date_str # Return original string if parsing fails + + @staticmethod + def _convert_to_time_string(time_str: str) -> str: + try: + time_obj = datetime.strptime(time_str, "%H:%M:%S").time() + return f"time({time_obj.hour}, {time_obj.minute}, {time_obj.second})" + except ValueError: + return time_str # Return original string if parsing fails + def generate_model( self, table: TableMeta, @@ -222,8 +280,15 @@ def create_header(self, *args, **kwargs) -> str: header = "" if self.uuid_import: header += pt.uuid_import + "\n" - if self.datetime_import: - header += pt.datetime_import + "\n" + if self.datetime_import or self.date_import or self.time_import: + imports = [] + if self.datetime_import: + imports.append("datetime") + if self.date_import: + imports.append("date") + if self.time_import: + imports.append("time") + header += f"from datetime import {', '.join(imports)}\n" if self.typing_imports: _imports = list(self.typing_imports) _imports.sort() diff --git a/omymodels/models/pydantic/types.py b/omymodels/models/pydantic/types.py index 3ee1d45..7f59790 100644 --- a/omymodels/models/pydantic/types.py +++ b/omymodels/models/pydantic/types.py @@ -1,3 +1,9 @@ from omymodels.types import mapper, populate_types_mapping -types_mapping = populate_types_mapping(mapper) +direct_types = { + ("date",): "date", + ("time",): "time", + ("year",): "int", +} + +types_mapping = populate_types_mapping({**mapper, **direct_types}) From e5433229f9e108de9bd34f3de71641181a686abd Mon Sep 17 00:00:00 2001 From: Ray Li Date: Mon, 30 Sep 2024 23:42:52 -0500 Subject: [PATCH 22/24] Create Date and Time Types #2 --- omymodels/models/pydantic/core.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/omymodels/models/pydantic/core.py b/omymodels/models/pydantic/core.py index 4f1f6b7..764166e 100644 --- a/omymodels/models/pydantic/core.py +++ b/omymodels/models/pydantic/core.py @@ -148,19 +148,15 @@ def get_default_value(self, column: Column) -> str: return column.default @staticmethod - def get_default_value_string(column: Column) -> str: + def get_default_value_string(self, column: Column) -> str: # Handle datetime default values if column.type.lower() in ["datetime", "timestamp"]: if datetime_now_check(column.default.lower()): column.default = "datetime.now()" elif column.type.lower() == "date": - column.default = ModelGenerator._convert_to_date_string( - column.default.strip("'") - ) + column.default = self._convert_to_date_string(column.default.strip("'")) elif column.type.lower() == "time": - column.default = ModelGenerator._convert_to_time_string( - column.default.strip("'") - ) + column.default = self._convert_to_time_string(column.default.strip("'")) # If the default is 'NULL', don't set a default in Pydantic (it already defaults to None) if column.default.lower() == "null": @@ -226,7 +222,7 @@ def _generate_valid_identifier(self, name: str) -> str: return valid_name - @staticmethod + @classmethod def _convert_to_date_string(date_str: str) -> str: try: date_obj = datetime.strptime(date_str, "%Y-%m-%d").date() @@ -234,7 +230,7 @@ def _convert_to_date_string(date_str: str) -> str: except ValueError: return date_str # Return original string if parsing fails - @staticmethod + @classmethod def _convert_to_time_string(time_str: str) -> str: try: time_obj = datetime.strptime(time_str, "%H:%M:%S").time() From f9d4aa5fa7f3121c36359ce527c6edef070451eb Mon Sep 17 00:00:00 2001 From: Ray Li Date: Mon, 30 Sep 2024 23:47:59 -0500 Subject: [PATCH 23/24] Create Date and Time Types #3 --- omymodels/models/pydantic/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omymodels/models/pydantic/core.py b/omymodels/models/pydantic/core.py index 764166e..b438817 100644 --- a/omymodels/models/pydantic/core.py +++ b/omymodels/models/pydantic/core.py @@ -147,7 +147,7 @@ def get_default_value(self, column: Column) -> str: return column.default - @staticmethod + @classmethod def get_default_value_string(self, column: Column) -> str: # Handle datetime default values if column.type.lower() in ["datetime", "timestamp"]: From 052bc4000ae7413e8716f271b3cb3501b6532d37 Mon Sep 17 00:00:00 2001 From: Ray Li Date: Mon, 30 Sep 2024 23:55:22 -0500 Subject: [PATCH 24/24] Fix Annotations --- omymodels/models/pydantic/core.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/omymodels/models/pydantic/core.py b/omymodels/models/pydantic/core.py index b438817..c1f8ef5 100644 --- a/omymodels/models/pydantic/core.py +++ b/omymodels/models/pydantic/core.py @@ -147,7 +147,6 @@ def get_default_value(self, column: Column) -> str: return column.default - @classmethod def get_default_value_string(self, column: Column) -> str: # Handle datetime default values if column.type.lower() in ["datetime", "timestamp"]: @@ -176,7 +175,6 @@ def get_default_value_string(self, column: Column) -> str: # Append the default value if it's not None (e.g., explicit default values like '0' or CURRENT_TIMESTAMP) return pt.pydantic_default_attr.format(default=default_value) - @classmethod def _is_valid_identifier(self, name: str) -> bool: return ( name.isidentifier() @@ -184,7 +182,6 @@ def _is_valid_identifier(self, name: str) -> bool: and not self._is_pydantic_reserved_name(name) ) - @classmethod def _is_pydantic_reserved_name(self, name: str) -> bool: """Check if the name is a Pydantic-specific reserved name or starts with a reserved prefix.""" pydantic_reserved_prefixes = {"dict_", "json_"} @@ -206,7 +203,6 @@ def _is_pydantic_reserved_name(self, name: str) -> bool: or name in pydantic_reserved_names ) - @classmethod def _generate_valid_identifier(self, name: str) -> str: """Generate a valid Python identifier from a given name.""" # Replace non-alphanumeric characters with underscores @@ -222,16 +218,14 @@ def _generate_valid_identifier(self, name: str) -> str: return valid_name - @classmethod - def _convert_to_date_string(date_str: str) -> str: + def _convert_to_date_string(self, date_str: str) -> str: try: date_obj = datetime.strptime(date_str, "%Y-%m-%d").date() return f"date({date_obj.year}, {date_obj.month}, {date_obj.day})" except ValueError: return date_str # Return original string if parsing fails - @classmethod - def _convert_to_time_string(time_str: str) -> str: + def _convert_to_time_string(self, time_str: str) -> str: try: time_obj = datetime.strptime(time_str, "%H:%M:%S").time() return f"time({time_obj.hour}, {time_obj.minute}, {time_obj.second})"