diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 06aa8db..e089081 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -4,28 +4,28 @@ Contains all of the Fides resources modeled as Pydantic models. """ from __future__ import annotations + from datetime import datetime from enum import Enum from typing import Any, Dict, List, Optional, Union -from typing_extensions import Annotated from packaging.version import InvalidVersion, Version - - from pydantic import ( - BeforeValidator, - field_validator, - model_validator, - ConfigDict, AnyUrl, BaseModel, + BeforeValidator, + ConfigDict, Field, HttpUrl, PositiveInt, + field_validator, + model_validator, ) +from typing_extensions import Annotated from fideslang.validation import ( FidesKey, + FidesValidationError, deprecated_version_later_than_added, has_versioning_if_default, is_deprecated_if_replaced, @@ -34,18 +34,15 @@ parse_data_type_string, sort_list_objects_by_name, unique_items_in_list, - valid_data_type, validate_fides_key, FidesValidationError, + valid_data_type, + validate_fides_key, ) -matching_parent_key_validator = field_validator("parent_key")( - matching_parent_key +matching_parent_key_validator = field_validator("parent_key")(matching_parent_key) +no_self_reference_validator = field_validator("parent_key")(no_self_reference) +has_versioning_if_default_validator = field_validator("is_default")( + has_versioning_if_default ) -no_self_reference_validator = field_validator("parent_key")( - no_self_reference -) -has_versioning_if_default_validator = field_validator( - "is_default" -)(has_versioning_if_default) deprecated_version_later_than_added_validator = field_validator( "version_deprecated", )(deprecated_version_later_than_added) @@ -75,7 +72,9 @@ class FidesModel(BaseModel): description="Defines the Organization that this resource belongs to.", ) tags: Optional[List[str]] = None - name: Optional[str] = Field(default=None, description="Human-Readable name for this resource.") + name: Optional[str] = Field( + default=None, description="Human-Readable name for this resource." + ) description: Optional[str] = description_field model_config = ConfigDict(extra="ignore", from_attributes=True) @@ -111,9 +110,7 @@ class DefaultModel(BaseModel): @field_validator("version_added") @classmethod - def validate_version_added( - cls, version_added: Optional[str] - ) -> Optional[str]: + def validate_version_added(cls, version_added: Optional[str]) -> Optional[str]: """ Validate that the `version_added` field is a proper Version """ @@ -286,7 +283,8 @@ class DataSubjectRights(BaseModel): description="Defines the strategy used when mapping data rights to a data subject.", ) values: Optional[List[DataSubjectRightsEnum]] = Field( - default=None, description="A list of valid data subject rights to be used when applying data rights to a data subject via a strategy.", + default=None, + description="A list of valid data subject rights to be used when applying data rights to a data subject via a strategy.", ) @model_validator(mode="before") @@ -307,7 +305,9 @@ def include_exclude_has_values(cls, values: Dict) -> Dict: class DataSubject(FidesModel, DefaultModel): """The DataSubject resource model.""" - rights: Optional[DataSubjectRights] = Field(default=None, description=DataSubjectRights.__doc__) + rights: Optional[DataSubjectRights] = Field( + default=None, description=DataSubjectRights.__doc__ + ) automated_decisions_or_profiling: Optional[bool] = Field( default=None, description="A boolean value to annotate whether or not automated decisions/profiling exists for the data subject.", @@ -348,7 +348,8 @@ class MyDatasetField(DatasetFieldBase): name: str = name_field description: Optional[str] = description_field data_categories: Optional[List[FidesKey]] = Field( - default=None, description="Arrays of Data Categories, identified by `fides_key`, that applies to this field.", + default=None, + description="Arrays of Data Categories, identified by `fides_key`, that applies to this field.", ) @@ -375,25 +376,28 @@ class FidesMeta(BaseModel): default=None, ) identity: Optional[str] = Field( - default=None, description="The type of the identity data that should be used to query this collection for a DSR." + default=None, + description="The type of the identity data that should be used to query this collection for a DSR.", ) primary_key: Optional[bool] = Field( default=None, - description="Whether the current field can be considered a primary key of the current collection" + description="Whether the current field can be considered a primary key of the current collection", ) data_type: Optional[str] = Field( - default=None, description="Optionally specify the data type. Fides will attempt to cast values to this type when querying." + default=None, + description="Optionally specify the data type. Fides will attempt to cast values to this type when querying.", ) length: Optional[PositiveInt] = Field( - default=None, description="Optionally specify the allowable field length. Fides will not generate values that exceed this size." + default=None, + description="Optionally specify the allowable field length. Fides will not generate values that exceed this size.", ) return_all_elements: Optional[bool] = Field( default=None, - description="Optionally specify to query for the entire array if the array is an entrypoint into the node. Default is False." + description="Optionally specify to query for the entire array if the array is an entrypoint into the node. Default is False.", ) read_only: Optional[bool] = Field( default=None, - description="Optionally specify if a field is read-only, meaning it can't be updated or deleted." + description="Optionally specify if a field is read-only, meaning it can't be updated or deleted.", ) @field_validator("data_type") @@ -428,7 +432,8 @@ class DatasetField(DatasetFieldBase, FidesopsMetaBackwardsCompat): fides_meta: Optional[FidesMeta] = None fields: Optional[List[DatasetField]] = Field( - default=None, description="An optional array of objects that describe hierarchical/nested fields (typically found in NoSQL databases).", + default=None, + description="An optional array of objects that describe hierarchical/nested fields (typically found in NoSQL databases).", ) @field_validator("fides_meta") @@ -518,7 +523,8 @@ class DatasetCollection(FidesopsMetaBackwardsCompat): name: str = name_field description: Optional[str] = description_field data_categories: Optional[List[FidesKey]] = Field( - default=None, description="Array of Data Category resources identified by `fides_key`, that apply to all fields in the collection.", + default=None, + description="Array of Data Category resources identified by `fides_key`, that apply to all fields in the collection.", ) fields: List[DatasetField] = Field( description="An array of objects that describe the collection's fields.", @@ -526,12 +532,8 @@ class DatasetCollection(FidesopsMetaBackwardsCompat): fides_meta: Optional[CollectionMeta] = None - _sort_fields: classmethod = field_validator("fields")( - sort_list_objects_by_name - ) - _unique_items_in_list: classmethod = field_validator("fields")( - unique_items_in_list - ) + _sort_fields: classmethod = field_validator("fields")(sort_list_objects_by_name) + _unique_items_in_list: classmethod = field_validator("fields")(unique_items_in_list) class ContactDetails(BaseModel): @@ -579,7 +581,8 @@ class Dataset(FidesModel, FidesopsMetaBackwardsCompat): meta: Optional[Dict] = meta_field data_categories: Optional[List[FidesKey]] = Field( - default=None, description="Array of Data Category resources identified by `fides_key`, that apply to all collections in the Dataset.", + default=None, + description="Array of Data Category resources identified by `fides_key`, that apply to all collections in the Dataset.", ) fides_meta: Optional[DatasetMetadata] = Field( description=DatasetMetadata.__doc__, default=None @@ -673,7 +676,8 @@ class OrganizationMetadata(BaseModel): """ resource_filters: Optional[List[ResourceFilter]] = Field( - default=None, description="A list of filters that can be used when generating or scanning systems." + default=None, + description="A list of filters that can be used when generating or scanning systems.", ) @@ -690,16 +694,20 @@ class Organization(FidesModel): description="An inherited field from the FidesModel that is unused with an Organization.", ) controller: Optional[ContactDetails] = Field( - default=None, description=ContactDetails.__doc__, + default=None, + description=ContactDetails.__doc__, ) data_protection_officer: Optional[ContactDetails] = Field( - default=None, description=ContactDetails.__doc__, + default=None, + description=ContactDetails.__doc__, ) fidesctl_meta: Optional[OrganizationMetadata] = Field( - default=None, description=OrganizationMetadata.__doc__, + default=None, + description=OrganizationMetadata.__doc__, ) representative: Optional[ContactDetails] = Field( - default=None, description=ContactDetails.__doc__, + default=None, + description=ContactDetails.__doc__, ) security_policy: Optional[HttpUrl] = Field( default=None, description="Am optional URL to the organization security policy." @@ -765,9 +773,7 @@ class Policy(FidesModel): description=PolicyRule.__doc__, ) - _sort_rules: classmethod = field_validator("rules")( - sort_list_objects_by_name - ) + _sort_rules: classmethod = field_validator("rules")(sort_list_objects_by_name) class PrivacyDeclaration(BaseModel): @@ -779,7 +785,8 @@ class PrivacyDeclaration(BaseModel): """ name: Optional[str] = Field( - default=None, description="The name of the privacy declaration on the system.", + default=None, + description="The name of the privacy declaration on the system.", ) data_categories: List[FidesKey] = Field( description="An array of data categories describing a system in a privacy declaration.", @@ -792,13 +799,16 @@ class PrivacyDeclaration(BaseModel): description="An array of data subjects describing a system in a privacy declaration.", ) dataset_references: Optional[List[FidesKey]] = Field( - default=None, description="Referenced Dataset fides keys used by the system.", + default=None, + description="Referenced Dataset fides keys used by the system.", ) egress: Optional[List[FidesKey]] = Field( - default=None, description="The resources to which data is sent. Any `fides_key`s included in this list reference `DataFlow` entries in the `egress` array of any `System` resources to which this `PrivacyDeclaration` is applied." + default=None, + description="The resources to which data is sent. Any `fides_key`s included in this list reference `DataFlow` entries in the `egress` array of any `System` resources to which this `PrivacyDeclaration` is applied.", ) ingress: Optional[List[FidesKey]] = Field( - default=None, description="The resources from which data is received. Any `fides_key`s included in this list reference `DataFlow` entries in the `ingress` array of any `System` resources to which this `PrivacyDeclaration` is applied." + default=None, + description="The resources from which data is received. Any `fides_key`s included in this list reference `DataFlow` entries in the `ingress` array of any `System` resources to which this `PrivacyDeclaration` is applied.", ) features: List[str] = Field( default_factory=list, description="The features of processing personal data." @@ -808,34 +818,40 @@ class PrivacyDeclaration(BaseModel): default=True, ) legal_basis_for_processing: Optional[LegalBasisForProcessingEnum] = Field( - default=None, description="The legal basis under which personal data is processed for this purpose." + default=None, + description="The legal basis under which personal data is processed for this purpose.", ) impact_assessment_location: Optional[str] = Field( - default=None, description="Where the legitimate interest impact assessment is stored" + default=None, + description="Where the legitimate interest impact assessment is stored", ) retention_period: Optional[str] = Field( - default=None, description="An optional string to describe the time period for which data is retained for this purpose." + default=None, + description="An optional string to describe the time period for which data is retained for this purpose.", ) processes_special_category_data: bool = Field( default=False, description="This system processes special category data", ) special_category_legal_basis: Optional[SpecialCategoryLegalBasisEnum] = Field( - default=None, description="The legal basis under which the special category data is processed.", + default=None, + description="The legal basis under which the special category data is processed.", ) data_shared_with_third_parties: bool = Field( default=False, description="This system shares data with third parties for this purpose.", ) third_parties: Optional[str] = Field( - default=None, description="The types of third parties the data is shared with.", + default=None, + description="The types of third parties the data is shared with.", ) shared_categories: List[str] = Field( default_factory=list, description="The categories of personal data that this system shares with third parties.", ) cookies: Optional[List[Cookies]] = Field( - default=None, description="Cookies associated with this data use to deliver services and functionality", + default=None, + description="Cookies associated with this data use to deliver services and functionality", ) model_config = ConfigDict(from_attributes=True) @@ -848,13 +864,16 @@ class SystemMetadata(BaseModel): """ resource_id: Optional[str] = Field( - default=None, description="The external resource id for the system being modeled." + default=None, + description="The external resource id for the system being modeled.", ) endpoint_address: Optional[str] = Field( - default=None, description="The host of the external resource for the system being modeled." + default=None, + description="The host of the external resource for the system being modeled.", ) endpoint_port: Optional[str] = Field( - default=None, description="The port of the external resource for the system being modeled." + default=None, + description="The port of the external resource for the system being modeled.", ) @@ -884,7 +903,8 @@ class DataFlow(BaseModel): description=f"Specifies the resource model class for which the `fides_key` applies. May be any of {', '.join([member.value for member in FlowableResources])}.", ) data_categories: Optional[List[FidesKey]] = Field( - default=None, description="An array of data categories describing the data in transit.", + default=None, + description="An array of data categories describing the data in transit.", ) @model_validator(mode="before") @@ -924,7 +944,8 @@ class System(FidesModel): meta: Optional[Dict] = meta_field fidesctl_meta: Optional[SystemMetadata] = Field( - default=None, description=SystemMetadata.__doc__, + default=None, + description=SystemMetadata.__doc__, ) system_type: str = Field( description="A required value to describe the type of system being modeled, examples include: Service, Application, Third Party, etc.", @@ -943,13 +964,16 @@ class System(FidesModel): description="An optional value to identify the owning department or group of the system within your organization", ) vendor_id: Optional[str] = Field( - default=None, description="The unique identifier for the vendor that's associated with this system." + default=None, + description="The unique identifier for the vendor that's associated with this system.", ) previous_vendor_id: Optional[str] = Field( - default=None, description="If specified, the unique identifier for the vendor that was previously associated with this system." + default=None, + description="If specified, the unique identifier for the vendor that was previously associated with this system.", ) vendor_deleted_date: Optional[datetime] = Field( - default=None, description="The deleted date of the vendor that's associated with this system." + default=None, + description="The deleted date of the vendor that's associated with this system.", ) dataset_references: List[FidesKey] = Field( default_factory=list, @@ -964,7 +988,8 @@ class System(FidesModel): description="This toggle indicates whether the system is exempt from privacy regulation if they do process personal data.", ) reason_for_exemption: Optional[str] = Field( - default=None, description="The reason that the system is exempt from privacy regulation." + default=None, + description="The reason that the system is exempt from privacy regulation.", ) uses_profiling: bool = Field( default=False, @@ -990,16 +1015,20 @@ class System(FidesModel): default=None, description="Location where the DPAs or DIPAs can be found." ) dpa_progress: Optional[str] = Field( - default=None, description="The optional status of a Data Protection Impact Assessment" + default=None, + description="The optional status of a Data Protection Impact Assessment", ) privacy_policy: Optional[AnyUrl] = Field( - default=None, description="A URL that points to the system's publicly accessible privacy policy." + default=None, + description="A URL that points to the system's publicly accessible privacy policy.", ) legal_name: Optional[str] = Field( - default=None, description="The legal name for the business represented by the system." + default=None, + description="The legal name for the business represented by the system.", ) legal_address: Optional[str] = Field( - default=None, description="The legal address for the business represented by the system." + default=None, + description="The legal address for the business represented by the system.", ) responsibility: List[DataResponsibilityTitle] = Field( default_factory=list, @@ -1009,13 +1038,15 @@ class System(FidesModel): default=None, description="The official privacy contact address or DPO." ) joint_controller_info: Optional[str] = Field( - default=None, description="The party or parties that share the responsibility for processing personal data." + default=None, + description="The party or parties that share the responsibility for processing personal data.", ) data_security_practices: Optional[str] = Field( default=None, description="The data security practices employed by this system." ) cookie_max_age_seconds: Optional[int] = Field( - default=None, description="The maximum storage duration, in seconds, for cookies used by this system." + default=None, + description="The maximum storage duration, in seconds, for cookies used by this system.", ) uses_cookies: bool = Field( default=False, description="Whether this system uses cookie storage." @@ -1029,15 +1060,17 @@ class System(FidesModel): description="Whether the system uses non-cookie methods of storage or accessing information stored on a user's device.", ) legitimate_interest_disclosure_url: Optional[AnyUrl] = Field( - default=None, description="A URL that points to the system's publicly accessible legitimate interest disclosure." + default=None, + description="A URL that points to the system's publicly accessible legitimate interest disclosure.", ) cookies: Optional[List[Cookies]] = Field( - default=None, description="System-level cookies unassociated with a data use to deliver services and functionality", + default=None, + description="System-level cookies unassociated with a data use to deliver services and functionality", ) - _sort_privacy_declarations: classmethod = field_validator( - "privacy_declarations" - )(sort_list_objects_by_name) + _sort_privacy_declarations: classmethod = field_validator("privacy_declarations")( + sort_list_objects_by_name + ) @model_validator(mode="before") @classmethod diff --git a/src/fideslang/validation.py b/src/fideslang/validation.py index f3152fc..7c04d1b 100644 --- a/src/fideslang/validation.py +++ b/src/fideslang/validation.py @@ -4,10 +4,10 @@ import re from collections import Counter from typing import Dict, List, Optional, Pattern, Set, Tuple -from typing_extensions import Annotated from packaging.version import Version from pydantic import BeforeValidator, ValidationInfo +from typing_extensions import Annotated FIDES_KEY_PATTERN = r"^[a-zA-Z0-9_.<>-]+$" @@ -87,7 +87,9 @@ def deprecated_version_later_than_added( version_added: Optional[str] = values.data.get("version_added") # Convert into Versions - transformed_version_added: Version = Version(version_added) if version_added else Version("0") + transformed_version_added: Version = ( + Version(version_added) if version_added else Version("0") + ) transformed_version_deprecated: Version = Version(version_deprecated) if transformed_version_deprecated < transformed_version_added: diff --git a/tests/conftest.py b/tests/conftest.py index 8994e3d..91e306c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,7 @@ import pytest import yaml + from fideslang import models diff --git a/tests/fideslang/gvl/test_gvl.py b/tests/fideslang/gvl/test_gvl.py index abda6fd..633d13b 100644 --- a/tests/fideslang/gvl/test_gvl.py +++ b/tests/fideslang/gvl/test_gvl.py @@ -67,19 +67,15 @@ def test_feature_id_to_feature_name(): assert feature_id_to_feature_name(feature_id=1001) is None - def test_data_category_id_to_data_categories(): - assert data_category_id_to_data_categories(1) == [ - "user.device.ip_address" - ] + assert data_category_id_to_data_categories(1) == ["user.device.ip_address"] # let's test one other data category just to be comprehensive assert data_category_id_to_data_categories(5) == [ - "user.account", - "user.unique_id", - "user.device" - ] - + "user.account", + "user.unique_id", + "user.device", + ] # assert invalid categories raise KeyErrors with pytest.raises(KeyError): diff --git a/tests/fideslang/test_parse.py b/tests/fideslang/test_parse.py index d8e7517..5c90cd7 100644 --- a/tests/fideslang/test_parse.py +++ b/tests/fideslang/test_parse.py @@ -1,7 +1,6 @@ import pytest -from fideslang import models -from fideslang import parse +from fideslang import models, parse @pytest.mark.unit diff --git a/tests/fideslang/test_validation.py b/tests/fideslang/test_validation.py index c17d7d7..e888b6e 100644 --- a/tests/fideslang/test_validation.py +++ b/tests/fideslang/test_validation.py @@ -21,7 +21,12 @@ PrivacyRule, System, ) -from fideslang.validation import FidesKey, FidesValidationError, valid_data_type, validate_fides_key +from fideslang.validation import ( + FidesKey, + FidesValidationError, + valid_data_type, + validate_fides_key, +) DEFAULT_TAXONOMY_CLASSES = [DataCategory, DataUse, DataSubject] @@ -458,6 +463,7 @@ def test_fides_key_validate_bad_key(): with pytest.raises(FidesValidationError): validate_fides_key("hi!") + @pytest.mark.unit def test_fides_key_validate_good_key(): validate_fides_key("hello_test_file.txt")