From c63010a78c0c3a3ef13b555fa581b90f8e3f4836 Mon Sep 17 00:00:00 2001 From: Khari Gardner Date: Mon, 4 Mar 2024 22:29:40 -0500 Subject: [PATCH 1/2] Updated falsy types in dataclass serialization --- poetry.lock | 29 +++++++++++++++- pyfivetran/__init__.py | 2 +- pyfivetran/client.py | 10 +++++- pyfivetran/endpoints/base.py | 3 ++ pyfivetran/endpoints/certificate.py | 44 ++++++++++++------------- pyfivetran/endpoints/connector.py | 51 ++++++++++++++--------------- pyfivetran/endpoints/destination.py | 12 ++++--- pyfivetran/endpoints/group.py | 12 +++---- pyfivetran/endpoints/roles.py | 2 +- pyfivetran/endpoints/schema.py | 10 +++--- pyfivetran/endpoints/users.py | 2 +- pyproject.toml | 2 ++ tests/endpoints/test_certificate.py | 12 +++---- tests/endpoints/test_connector.py | 3 +- 14 files changed, 119 insertions(+), 75 deletions(-) diff --git a/poetry.lock b/poetry.lock index 719d2c3..de04d8c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -472,6 +472,21 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] +[[package]] +name = "isort" +version = "5.13.2" +description = "A Python utility / library to sort Python imports." +category = "dev" +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, + {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, +] + +[package.extras] +colors = ["colorama (>=0.4.6)"] + [[package]] name = "jinja2" version = "3.1.2" @@ -1322,6 +1337,18 @@ dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "pre-commit (>=2 doc = ["cairosvg (>=2.5.2,<3.0.0)", "mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "pillow (>=9.3.0,<10.0.0)"] test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.910)", "pytest (>=4.4.0,<8.0.0)", "pytest-cov (>=2.10.0,<5.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<4.0.0)", "rich (>=10.11.0,<14.0.0)", "shellingham (>=1.3.0,<2.0.0)"] +[[package]] +name = "types-python-dateutil" +version = "2.8.19.20240106" +description = "Typing stubs for python-dateutil" +category = "dev" +optional = false +python-versions = ">=3.8" +files = [ + {file = "types-python-dateutil-2.8.19.20240106.tar.gz", hash = "sha256:1f8db221c3b98e6ca02ea83a58371b22c374f42ae5bbdf186db9c9a76581459f"}, + {file = "types_python_dateutil-2.8.19.20240106-py3-none-any.whl", hash = "sha256:efbbdc54590d0f16152fa103c9879c7d4a00e82078f6e2cf01769042165acaa2"}, +] + [[package]] name = "typing-extensions" version = "4.8.0" @@ -1391,4 +1418,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.8, <3.12" -content-hash = "95f8011704d6648ed771fb05d87c2865d79f347cab052c1b20ab70f38681c431" +content-hash = "6c652e62841418cd21d2eda43e4f1f8580ecbbefda430cb50ec89b6fe3272895" diff --git a/pyfivetran/__init__.py b/pyfivetran/__init__.py index 862d331..8192b99 100644 --- a/pyfivetran/__init__.py +++ b/pyfivetran/__init__.py @@ -1 +1 @@ -from .endpoints import * # noqa +from .client import FivetranClient \ No newline at end of file diff --git a/pyfivetran/client.py b/pyfivetran/client.py index d9baa6f..c008790 100644 --- a/pyfivetran/client.py +++ b/pyfivetran/client.py @@ -16,7 +16,7 @@ UserEndpoint, WebhookEndpoint, ) - +from pyfivetran.shed import GeneralApiResponse class AuthenticationTuple(NamedTuple): basic: httpx.BasicAuth @@ -35,6 +35,14 @@ def __init__(self, api_key: str, api_secret: str) -> None: self.api_secret = api_secret self._client = httpx.Client() + @lazy + def account_info(self) -> GeneralApiResponse: + """ + Returns information about current account from API key. + """ + url = '"https://api.fivetran.com/v1/account/info"' + return self.client.get(url).json() + @property def authentication(self) -> AuthenticationTuple: return AuthenticationTuple( diff --git a/pyfivetran/endpoints/base.py b/pyfivetran/endpoints/base.py index 5b32e09..3a611e5 100644 --- a/pyfivetran/endpoints/base.py +++ b/pyfivetran/endpoints/base.py @@ -76,3 +76,6 @@ class ApiDataclass(ABC): @abstractmethod def _from_dict(cls, endpoint, d: Dict[str, Any]) -> "ApiDataclass": ... + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(endpoint={self.endpoint.__class__.__name__}, fivetran_id={getattr(self, 'fivetran_id', None) or 'None'})" \ No newline at end of file diff --git a/pyfivetran/endpoints/certificate.py b/pyfivetran/endpoints/certificate.py index 0cab5e5..767ae62 100644 --- a/pyfivetran/endpoints/certificate.py +++ b/pyfivetran/endpoints/certificate.py @@ -54,7 +54,7 @@ def approve_certificate( def approve_fingerprint( self, - hash: str, + hash_: str, public_key: str | bytes, connector_id: Optional[str] = None, destination_id: Optional[str] = None, @@ -73,7 +73,7 @@ def approve_fingerprint( # just have to serialize it as a string for the API public_key = str(public_key) - payload = {"hash": hash, "public_key": public_key} + payload = {"hash": hash_, "public_key": public_key} if connector_id: endpoint_ = f"/connectors/{connector_id}/fingerprints" @@ -88,7 +88,7 @@ def approve_fingerprint( def get_certificate_details( self, - hash: str | bytes, + hash_: str | bytes, connector_id: Optional[str] = None, destination_id: Optional[str] = None, ) -> GeneralApiResponse: @@ -100,14 +100,14 @@ def get_certificate_details( :param connector_id: Unique ID of the connector in Fivetran :return: GeneralApiResponse """ - if isinstance(hash, bytes): + if isinstance(hash_, bytes): # just have to serialize it as a string for the API - hash = str(hash) + hash_ = str(hash_) if connector_id: - endpoint_ = f"/connectors/{connector_id}/certificates/{hash}" + endpoint_ = f"/connectors/{connector_id}/certificates/{hash_}" elif destination_id: - endpoint_ = f"/destinations/{destination_id}/certificates/{hash}" + endpoint_ = f"/destinations/{destination_id}/certificates/{hash_}" else: raise ValueError("Either connector_id or destination_id must be provided") @@ -115,7 +115,7 @@ def get_certificate_details( def get_fingerprint_details( self, - hash: str | bytes, + hash_: str | bytes, connector_id: Optional[str] = None, destination_id: Optional[str] = None, ) -> GeneralApiResponse: @@ -127,14 +127,14 @@ def get_fingerprint_details( :param connector_id: Unique ID of the connector in Fivetran :return: GeneralApiResponse """ - if isinstance(hash, bytes): + if isinstance(hash_, bytes): # just have to serialize it as a string for the API - hash = str(hash) + hash_ = str(hash_) if connector_id: - endpoint_ = f"/connectors/{connector_id}/fingerprints/{hash}" + endpoint_ = f"/connectors/{connector_id}/fingerprints/{hash_}" elif destination_id: - endpoint_ = f"/destinations/{destination_id}/fingerprints/{hash}" + endpoint_ = f"/destinations/{destination_id}/fingerprints/{hash_}" else: raise ValueError("Either connector_id or destination_id must be provided") @@ -218,7 +218,7 @@ def get_certificates( def revoke_certificate( self, - hash: str | bytes, + hash_: str | bytes, connector_id: Optional[str] = None, destination_id: Optional[str] = None, ) -> GeneralApiResponse: @@ -230,14 +230,14 @@ def revoke_certificate( :param connector_id: Unique ID of the connector in Fivetran :return: GeneralApiResponse """ - if isinstance(hash, bytes): + if isinstance(hash_, bytes): # just have to serialize it as a string for the API - hash = str(hash) + hash_ = str(hash_) if connector_id: - endpoint_ = f"/connectors/{connector_id}/certificates/{hash}" + endpoint_ = f"/connectors/{connector_id}/certificates/{hash_}" elif destination_id: - endpoint_ = f"/destinations/{destination_id}/certificates/{hash}" + endpoint_ = f"/destinations/{destination_id}/certificates/{hash_}" else: raise ValueError("Either connector_id or destination_id must be provided") @@ -245,7 +245,7 @@ def revoke_certificate( def revoke_fingerprint( self, - hash: str | bytes, + hash_: str | bytes, connector_id: Optional[str] = None, destination_id: Optional[str] = None, ) -> GeneralApiResponse: @@ -257,14 +257,14 @@ def revoke_fingerprint( :param connector_id: Unique ID of the connector in Fivetran :return: GeneralApiResponse """ - if isinstance(hash, bytes): + if isinstance(hash_, bytes): # just have to serialize it as a string for the API - hash = str(hash) + hash_ = str(hash_) if connector_id: - endpoint_ = f"/connectors/{connector_id}/fingerprints/{hash}" + endpoint_ = f"/connectors/{connector_id}/fingerprints/{hash_}" elif destination_id: - endpoint_ = f"/destinations/{destination_id}/fingerprints/{hash}" + endpoint_ = f"/destinations/{destination_id}/fingerprints/{hash_}" else: raise ValueError("Either connector_id or destination_id must be provided") diff --git a/pyfivetran/endpoints/connector.py b/pyfivetran/endpoints/connector.py index 60348b4..899ab5b 100644 --- a/pyfivetran/endpoints/connector.py +++ b/pyfivetran/endpoints/connector.py @@ -6,6 +6,7 @@ from httpx import HTTPStatusError +from pyfivetran.utils import deserialize_timestamp from pyfivetran.endpoints.base import Endpoint, Client, ApiDataclass from pyfivetran.shed import ( GeneralApiResponse, @@ -15,9 +16,6 @@ PaginatedApiResponse, ) -if TYPE_CHECKING: - pass - @dataclass class Connector(ApiDataclass): @@ -32,6 +30,7 @@ class Connector(ApiDataclass): service_version: int created_at: datetime | str data_delay_senstivity: Literal["LOW", "NORMAL", "HIGH", "CUSTOM"] = "NORMAL" + setup_tests: Optional[List[Dict[str, Any]]] = None source_sync_details: Optional[Dict[str, Any]] = None data_delay_threshold: Optional[int] = 0 @@ -79,7 +78,7 @@ def modify( ]: raise ApiError("Invalid sync_frequency value provided") from ValueError() - payload: Dict[str, Any] = dict() + payload: Dict[str, Any] = {} if config is not None: payload["config"] = config @@ -129,14 +128,14 @@ def modify_state(self, state: Dict[str, Any]) -> GeneralApiResponse: method="PATCH", url=f"{self.as_url}/state", json=state ).json() - def delete(self) -> GeneralApiResponse: + def delete(self) -> Optional[GeneralApiResponse]: """ Deletes the connector. :return: GeneralApiResponse """ # TODO: need to adjust this to use the endpoint attribute + obj = self.endpoint.client.delete(url=self.as_url) try: - obj = self.endpoint.client.delete(url=self.as_url) obj.raise_for_status() obj_json = obj.json() self._is_deleted = True @@ -170,7 +169,7 @@ def run_setup_tests( :param trust_fingerprints: Whether to trust fingerprints :return: GeneralApiResponse """ - payload = dict() + payload = {} if trust_certificates is not None: payload["trust_certificates"] = trust_certificates @@ -211,40 +210,40 @@ def _from_dict(cls, endpoint, d: Dict[str, Any]) -> "Connector": # convert to datetimes # timestamps come in the format: 2019-08-24T14:15:22Z if d.get("succeeded_at") and isinstance(d.get("succeeded_at"), str): - d["succeeded_at"] = datetime.strptime( - d.get("succeeded_at"), "%Y-%m-%dT%H:%M:%SZ" - ) # type: ignore + d["succeeded_at"] = deserialize_timestamp( + d["succeeded_at"] + ) if d.get("created_at") and isinstance(d.get("created_at"), str): - d["created_at"] = datetime.strptime( - d.get("created_at"), "%Y-%m-%dT%H:%M:%SZ" - ) # type: ignore + d["created_at"] = deserialize_timestamp( + d["created_at"] + ) if d.get("failed_at") and isinstance(d.get("failed_at"), str): - d["failed_at"] = datetime.strptime(d.get("failed_at"), "%Y-%m-%dT%H:%M:%SZ") # type: ignore + d["failed_at"] = deserialize_timestamp(d["failed_at"]) cls_to_return = cls( - fivetran_id=d.get("id"), # type: ignore - service=d.get("service"), # type: ignore - schema=d.get("schema"), # type: ignore - paused=d.get("paused"), # type: ignore + fivetran_id=d['id'], + service=d['service'], + schema=d['schema'], + paused=d['paused'], status=d.get("status"), daily_sync_time=d.get("daily_sync_time"), succeeded_at=d.get("succeeded_at"), connect_card=d.get("connect_card"), - sync_frequency=d.get("sync_frequency"), # type: ignore - pause_after_trial=d.get("pause_after_trial"), # type: ignore + sync_frequency=int(d["sync_frequency"]), + pause_after_trial=bool(d["pause_after_trial"]), data_delay_threshold=d.get("data_delay_threshold"), - group_id=d.get("group_id"), # type: ignore - connected_by=d.get("connected_by"), # type: ignore + group_id=str(d["group_id"]), + connected_by=d["connected_by"], setup_tests=d.get("setup_tests"), source_sync_details=d.get("source_sync_details"), - service_version=d.get("service_version"), # type: ignore - created_at=d.get("created_at"), # type: ignore + service_version=d["service_version"], + created_at=d["created_at"], failed_at=d.get("failed_at"), - schedule_type=d.get("schedule_type"), # type: ignore + schedule_type=d["schedule_type"], connect_card_config=d.get("connect_card_config"), config=d.get("config"), _is_deleted=False, - endpoint=endpoint, + endpoint=endpoint ) setattr(cls_to_return, "_raw", d) diff --git a/pyfivetran/endpoints/destination.py b/pyfivetran/endpoints/destination.py index 8407ea9..3e9e102 100644 --- a/pyfivetran/endpoints/destination.py +++ b/pyfivetran/endpoints/destination.py @@ -35,7 +35,7 @@ def as_url(self) -> str: @property def raw(self) -> Dict[str, Any]: - return self._raw if hasattr(self, "_raw") else self.__dict__ + return getattr(self, "_raw", self.__dict__) def delete(self) -> GeneralApiResponse: resp = self.endpoint._request(method="DELETE", url=self.as_url).json() @@ -71,7 +71,7 @@ def modify( else: region = region - payload: Dict[str, Any] = dict() + payload: Dict[str, Any] = {} if region is not None: payload["region"] = region @@ -109,7 +109,7 @@ def run_setup_tests( :param trust_fingerprints: Whether to trust fingerprints :return: GeneralApiResponse """ - payload = dict() + payload = {} if trust_certificates is not None: payload["trust_certificates"] = trust_certificates @@ -194,9 +194,13 @@ def create_destination( """ if isinstance(region, str): try: - region = Region(region) + Region(region.upper()) except ValueError as e: raise ValueError(f"Invalid region: {region}") from e + elif isinstance(region, Region): + region = region.name.lower() + else: + region = None if isinstance(time_zone_offset, (str, tzinfo)): time_zone_offset = serialize_timezone(time_zone_offset) diff --git a/pyfivetran/endpoints/group.py b/pyfivetran/endpoints/group.py index 24d4a2b..a193ccc 100644 --- a/pyfivetran/endpoints/group.py +++ b/pyfivetran/endpoints/group.py @@ -28,7 +28,7 @@ def as_url(self) -> str: @property def raw(self) -> Dict[str, Any]: - return self._raw if hasattr(self, "_raw") else self.__dict__ + return getattr(self, "_raw", self.__dict__) @property def public_key(self) -> str: @@ -106,7 +106,7 @@ def add_user( :param role: The role of the user to add :return: GeneralApiResponse """ - payload = dict() + payload = {} if not email and not role: raise ValueError("Either email or role must be provided") @@ -124,7 +124,7 @@ def add_user( # TODO: change this to serialize to the Connector dataclass def list_connectors( self, schema: Optional[str] = None, limit: Optional[int] = None - ) -> List[PaginatedApiResponse]: + ) -> List[PaginatedApiResponse]: # sourcery skip: class-extract-method """ Returns a list of connectors in a group in your Fivetran account. @@ -132,7 +132,7 @@ def list_connectors( :param limit: The number of records to return :return: List[PaginatedApiResponse] """ - params: Dict[str, str | int] = dict() + params: Dict[str, str | int] = {} if schema is not None: params["schema"] = schema @@ -158,7 +158,7 @@ def list_users(self, limit: Optional[int] = None) -> List[PaginatedApiResponse]: :param limit: The number of records to return :return: List[PaginatedApiResponse] """ - params = dict() + params = {} if limit is not None: params["limit"] = limit @@ -215,7 +215,7 @@ def list_groups(self, limit: Optional[int] = None) -> List[Group]: :param limit: The number of records to return :return: List[PaginatedApiResponse] """ - params = dict() + params = {} if limit is not None: params["limit"] = limit diff --git a/pyfivetran/endpoints/roles.py b/pyfivetran/endpoints/roles.py index 67759df..0f211f2 100644 --- a/pyfivetran/endpoints/roles.py +++ b/pyfivetran/endpoints/roles.py @@ -20,7 +20,7 @@ def list_roles(self, limit: Optional[int] = None) -> List[PaginatedApiResponse]: :param limit: The number of records to return :return: List[PaginatedApiResponse] """ - params = dict() + params = {} if limit is not None: params["limit"] = limit diff --git a/pyfivetran/endpoints/schema.py b/pyfivetran/endpoints/schema.py index 7888109..3f4bc7c 100644 --- a/pyfivetran/endpoints/schema.py +++ b/pyfivetran/endpoints/schema.py @@ -34,7 +34,7 @@ def modify_connector_column_config( :param hashed: Whether the column is hashed :return: GeneralApiResponse """ - payload = dict() + payload = {} if enabled is not None: payload["enabled"] = enabled @@ -68,7 +68,7 @@ def modify_connector_database_schema_config( :param tables: The tables within the schema :return: GeneralApiResponse """ - payload: Dict[Any, Any] = dict() + payload: Dict[Any, Any] = {} if enabled is not None: payload["enabled"] = enabled @@ -110,7 +110,7 @@ def modify_connector_schema_config( "Invalid schema_change_handling value provided" ) from ValueError() - payload: Dict[Any, Any] = dict() + payload: Dict[Any, Any] = {} if schemas is not None: payload["schemas"] = schemas @@ -152,7 +152,7 @@ def modify_table_config( if sync_mode not in ["SOFT_DELETE", "HISTORY", "LIVE"]: raise ApiError("Invalid sync_mode value provided") from ValueError() - payload: Dict[Any, Any] = dict() + payload: Dict[Any, Any] = {} if enabled is not None: payload["enabled"] = enabled @@ -201,7 +201,7 @@ def reload_connector_schema_config( :return: GeneralApiResponse """ - payload: Dict[Any, Any] = dict() + payload: Dict[Any, Any] = {} if exclude_mode is not None: payload["exclude_mode"] = exclude_mode diff --git a/pyfivetran/endpoints/users.py b/pyfivetran/endpoints/users.py index 26474b5..0aecd30 100644 --- a/pyfivetran/endpoints/users.py +++ b/pyfivetran/endpoints/users.py @@ -38,7 +38,7 @@ def as_url(self) -> str: @property def raw(self) -> Dict[str, Any]: - return self._raw if hasattr(self, "_raw") else self.__dict__ + return getattr(self, "_raw", self.__dict__) @property def connector_memberships(self) -> Sequence[PaginatedApiResponse]: diff --git a/pyproject.toml b/pyproject.toml index 891a54f..42a2f04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,8 @@ pre-commit = "^3.5.0" pytest-mock = "^3.12.0" coverage = {extras = ["toml"], version = "^7.3.4"} pytest-cov = "^4.1.0" +isort = "^5.13.2" +types-python-dateutil = "^2.8.19.20240106" [tool.poetry.group.docs.dependencies] diff --git a/tests/endpoints/test_certificate.py b/tests/endpoints/test_certificate.py index 2dd2345..170c15e 100644 --- a/tests/endpoints/test_certificate.py +++ b/tests/endpoints/test_certificate.py @@ -80,7 +80,7 @@ def json(self): # Call the approve_fingerprint method with connector_id response = endpoint.approve_fingerprint( - hash="12345", public_key="abcde", connector_id="connector_123" + hash_="12345", public_key="abcde", connector_id="connector_123" ) # Assert that the _request method was called with the correct parameters @@ -121,7 +121,7 @@ def json(self): # Call the get_certificate_details method with connector_id response = endpoint.get_certificate_details( - hash="12345", connector_id="connector_123" + hash_="12345", connector_id="connector_123" ) # Assert that the _request method was called with the correct parameters @@ -138,7 +138,7 @@ def json(self): # Call the get_certificate_details method with destination_id response = endpoint.get_certificate_details( - hash="12345", destination_id="destination_123" + hash_="12345", destination_id="destination_123" ) # Assert that the _request method was called with the correct parameters @@ -160,15 +160,15 @@ def test_value_error_no_connector_id_or_destination_id(self, mocker): # Call the approve_fingerprint method without connector_id or destination_id with pytest.raises(ValueError): - endpoint.approve_fingerprint(hash="12345", public_key="abcde") + endpoint.approve_fingerprint(hash_="12345", public_key="abcde") # Call the get_certificate_details method without connector_id or destination_id with pytest.raises(ValueError): - endpoint.get_certificate_details(hash="12345") + endpoint.get_certificate_details(hash_="12345") # Call the get_fingerprint_details method without connector_id or destination_id with pytest.raises(ValueError): - endpoint.get_fingerprint_details(hash="12345") + endpoint.get_fingerprint_details(hash_="12345") # Call the get_fingerprints method without connector_id or destination_id with pytest.raises(ValueError): diff --git a/tests/endpoints/test_connector.py b/tests/endpoints/test_connector.py index 5efef9c..f493004 100644 --- a/tests/endpoints/test_connector.py +++ b/tests/endpoints/test_connector.py @@ -215,7 +215,7 @@ def test_get_connector(self, mocker): "code": 200, "message": "Success", "data": { - "fivetran_id": "12345", + "id": "12345", "service": "example_service", "schema": "example_schema", "paused": False, @@ -225,6 +225,7 @@ def test_get_connector(self, mocker): "connected_by": "user_123", "service_version": 1, "created_at": "2022-01-01T00:00:00Z", + "schedule_type": "manual", }, } From 7b8dd3a8b43082a0f91bcd91f5f9f9c5e7e4d08b Mon Sep 17 00:00:00 2001 From: Khari Gardner Date: Mon, 4 Mar 2024 22:31:58 -0500 Subject: [PATCH 2/2] Version bump --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 42a2f04..dbdb33c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pyfivetran" -version = "0.1.3" +version = "2024.03.04.02" description = "Pythonic interface to the fivetran API" authors = ["Khari Gardner "] license = "MIT"