diff --git a/ndonnx/_logic_in_data/_array.py b/ndonnx/_logic_in_data/_array.py index 39f430b..a000068 100644 --- a/ndonnx/_logic_in_data/_array.py +++ b/ndonnx/_logic_in_data/_array.py @@ -106,11 +106,14 @@ def unwrap_numpy(self) -> np.ndarray: """ return self._data.unwrap_numpy() - def to_numpy(self) -> np.ndarray: + def to_numpy(self) -> np.ndarray | None: from warnings import warn warn("'to_numpy' is deprecated in favor of 'unwrap_numpy'", DeprecationWarning) - return self.unwrap_numpy() + try: + return self.unwrap_numpy() + except ValueError: + return None def __getitem__(self, key: GetitemIndex, /) -> Array: from ._typed_array import TyArrayBool, TyArrayInteger diff --git a/ndonnx/_logic_in_data/_build.py b/ndonnx/_logic_in_data/_build.py index f68b1c9..90fb889 100644 --- a/ndonnx/_logic_in_data/_build.py +++ b/ndonnx/_logic_in_data/_build.py @@ -18,13 +18,16 @@ def build(arguments: dict[str, Array], results: dict[str, Array]) -> onnx.ModelP mp = spox_build(ins, outs, drop_unused_inputs=True) metadata = { - "schemas": json.dumps( + "ndonnx": json.dumps( { - "arguments": json.loads(_json_schema(arguments)), - "results": json.loads(_json_schema(results)), + "schemas": { + "arguments": json.loads(_json_schema(arguments)), + "results": json.loads(_json_schema(results)), + }, + # Version for how we organize our metadata in general + "metadata_version": "1", } - ), - "ndonnx_schema_version": "1", + ) } onnx.helper.set_model_props(mp, metadata) diff --git a/ndonnx/_logic_in_data/_schema.py b/ndonnx/_logic_in_data/_schema.py index 73425af..66e9656 100644 --- a/ndonnx/_logic_in_data/_schema.py +++ b/ndonnx/_logic_in_data/_schema.py @@ -4,38 +4,46 @@ import json from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Literal +from typing import Any, Literal import numpy as np import onnx - -if TYPE_CHECKING: - from spox import Var - from typing_extensions import Self - - Json = dict[str, "Json"] | list["Json"] | str | int | float | bool | None - PrimitiveComponent = Literal[ - "float16", - "float32", - "float64", - "int8", - "int16", - "int32", - "int64", - "uint8", - "uint16", - "uint32", - "uint64", - "bool", - "str", - ] - StructComponent = dict[str, "Schema"] - Components = dict[str, Var] +from spox import Var +from typing_extensions import Self + +Json = dict[str, "Json"] | list["Json"] | str | int | float | bool | None +"""A JSON serializable object.""" + +PrimitiveComponent = Literal[ + "float16", + "float32", + "float64", + "int8", + "int16", + "int32", + "int64", + "uint8", + "uint16", + "uint32", + "uint64", + "bool", + "str", +] +"""Primitive type with an equivalent type in the ONNX standard.""" + +StructComponent = dict[str, "Schema"] +"""A composite type consisting of other ``StructComponents`` or ``PrimitiveTypes``.""" + +Components = dict[str, Var] +"""A flattened representation of the components of an array with arbitrary data type.""" @dataclass class DTypeInfo: + """Class returned by ``DType._info`` describing the respective data type.""" + defining_library: str + # Version of this particular data type version: int dtype: str dtype_state: Json @@ -43,6 +51,11 @@ class DTypeInfo: @dataclass class Schema: + """Schema describing a data type. + + The names are suffixes of the names ultimately used in the model API. + """ + dtype_info: DTypeInfo components: PrimitiveComponent | StructComponent @@ -110,7 +123,15 @@ class Schemas: def get_schemas(mp: onnx.ModelProto) -> Schemas: metadict = {el.key: el.value for el in mp.metadata_props} - schema_dict = json.loads(metadict["schemas"]) + return _get_schemas(metadict) + + +def _get_schemas(metadata: dict[str, str]) -> Schemas: + """Get ``Schemas`` from metadata dict. + + This function is factored out from ``get_schemas`` for easier testing. + """ + schema_dict = json.loads(metadata["ndonnx"])["schemas"] arguments = {k: Schema.from_json(v, 1) for k, v in schema_dict["arguments"].items()} results = {k: Schema.from_json(v, 1) for k, v in schema_dict["results"].items()} diff --git a/ndonnx/_logic_in_data/_typed_array/onnx.py b/ndonnx/_logic_in_data/_typed_array/onnx.py index 9d5a864..cc3ed00 100644 --- a/ndonnx/_logic_in_data/_typed_array/onnx.py +++ b/ndonnx/_logic_in_data/_typed_array/onnx.py @@ -260,7 +260,7 @@ def unwrap_numpy(self) -> np.ndarray: # TODO: Where does the "object" kind come from? # Probably spox; we should have a more predictable # upstream string support. - return np_arr + return np_arr.astype(str) # Should not happen raise ValueError("unexpected value data type") diff --git a/tests/schemas/Bool.json b/tests/schemas/Bool.json new file mode 100644 index 0000000..a36118d --- /dev/null +++ b/tests/schemas/Bool.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/Float32.json b/tests/schemas/Float32.json new file mode 100644 index 0000000..ef9d247 --- /dev/null +++ b/tests/schemas/Float32.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Float32\", \"dtype_state\": null}, \"components\": \"float32\"}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Float32\", \"dtype_state\": null}, \"components\": \"float32\"}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/Float64.json b/tests/schemas/Float64.json new file mode 100644 index 0000000..c23a776 --- /dev/null +++ b/tests/schemas/Float64.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Float64\", \"dtype_state\": null}, \"components\": \"float64\"}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Float64\", \"dtype_state\": null}, \"components\": \"float64\"}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/Int16.json b/tests/schemas/Int16.json new file mode 100644 index 0000000..e299e6c --- /dev/null +++ b/tests/schemas/Int16.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Int16\", \"dtype_state\": null}, \"components\": \"int16\"}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Int16\", \"dtype_state\": null}, \"components\": \"int16\"}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/Int32.json b/tests/schemas/Int32.json new file mode 100644 index 0000000..cf39d74 --- /dev/null +++ b/tests/schemas/Int32.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Int32\", \"dtype_state\": null}, \"components\": \"int32\"}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Int32\", \"dtype_state\": null}, \"components\": \"int32\"}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/Int64.json b/tests/schemas/Int64.json new file mode 100644 index 0000000..b1d81bf --- /dev/null +++ b/tests/schemas/Int64.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Int64\", \"dtype_state\": null}, \"components\": \"int64\"}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Int64\", \"dtype_state\": null}, \"components\": \"int64\"}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/Int8.json b/tests/schemas/Int8.json new file mode 100644 index 0000000..dbf3489 --- /dev/null +++ b/tests/schemas/Int8.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Int8\", \"dtype_state\": null}, \"components\": \"int8\"}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Int8\", \"dtype_state\": null}, \"components\": \"int8\"}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/NBool.json b/tests/schemas/NBool.json new file mode 100644 index 0000000..d5188fd --- /dev/null +++ b/tests/schemas/NBool.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NBool\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NBool\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/NFloat32.json b/tests/schemas/NFloat32.json new file mode 100644 index 0000000..f772f68 --- /dev/null +++ b/tests/schemas/NFloat32.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NFloat32\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Float32\", \"dtype_state\": null}, \"components\": \"float32\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NFloat32\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Float32\", \"dtype_state\": null}, \"components\": \"float32\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/NFloat64.json b/tests/schemas/NFloat64.json new file mode 100644 index 0000000..67a1fe6 --- /dev/null +++ b/tests/schemas/NFloat64.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NFloat64\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Float64\", \"dtype_state\": null}, \"components\": \"float64\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NFloat64\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Float64\", \"dtype_state\": null}, \"components\": \"float64\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/NInt16.json b/tests/schemas/NInt16.json new file mode 100644 index 0000000..19d71a6 --- /dev/null +++ b/tests/schemas/NInt16.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NInt16\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Int16\", \"dtype_state\": null}, \"components\": \"int16\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NInt16\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Int16\", \"dtype_state\": null}, \"components\": \"int16\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/NInt32.json b/tests/schemas/NInt32.json new file mode 100644 index 0000000..b6cbe4d --- /dev/null +++ b/tests/schemas/NInt32.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NInt32\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Int32\", \"dtype_state\": null}, \"components\": \"int32\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NInt32\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Int32\", \"dtype_state\": null}, \"components\": \"int32\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/NInt64.json b/tests/schemas/NInt64.json new file mode 100644 index 0000000..a7b96fd --- /dev/null +++ b/tests/schemas/NInt64.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NInt64\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Int64\", \"dtype_state\": null}, \"components\": \"int64\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NInt64\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Int64\", \"dtype_state\": null}, \"components\": \"int64\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/NInt8.json b/tests/schemas/NInt8.json new file mode 100644 index 0000000..c37bf38 --- /dev/null +++ b/tests/schemas/NInt8.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NInt8\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Int8\", \"dtype_state\": null}, \"components\": \"int8\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NInt8\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Int8\", \"dtype_state\": null}, \"components\": \"int8\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/NString.json b/tests/schemas/NString.json new file mode 100644 index 0000000..c7b881f --- /dev/null +++ b/tests/schemas/NString.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NString\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"String\", \"dtype_state\": null}, \"components\": \"str\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NString\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"String\", \"dtype_state\": null}, \"components\": \"str\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/NUint16.json b/tests/schemas/NUint16.json new file mode 100644 index 0000000..fcb3819 --- /dev/null +++ b/tests/schemas/NUint16.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NUint16\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Uint16\", \"dtype_state\": null}, \"components\": \"uint16\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NUint16\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Uint16\", \"dtype_state\": null}, \"components\": \"uint16\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/NUint32.json b/tests/schemas/NUint32.json new file mode 100644 index 0000000..ebd04cf --- /dev/null +++ b/tests/schemas/NUint32.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NUint32\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Uint32\", \"dtype_state\": null}, \"components\": \"uint32\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NUint32\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Uint32\", \"dtype_state\": null}, \"components\": \"uint32\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/NUint64.json b/tests/schemas/NUint64.json new file mode 100644 index 0000000..fc0a738 --- /dev/null +++ b/tests/schemas/NUint64.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NUint64\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Uint64\", \"dtype_state\": null}, \"components\": \"uint64\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NUint64\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Uint64\", \"dtype_state\": null}, \"components\": \"uint64\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/NUint8.json b/tests/schemas/NUint8.json new file mode 100644 index 0000000..723124f --- /dev/null +++ b/tests/schemas/NUint8.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NUint8\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Uint8\", \"dtype_state\": null}, \"components\": \"uint8\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"NUint8\", \"dtype_state\": null}, \"components\": {\"data\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Uint8\", \"dtype_state\": null}, \"components\": \"uint8\"}, \"mask\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Bool\", \"dtype_state\": null}, \"components\": \"bool\"}}}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/String.json b/tests/schemas/String.json new file mode 100644 index 0000000..dd20a11 --- /dev/null +++ b/tests/schemas/String.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"String\", \"dtype_state\": null}, \"components\": \"str\"}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"String\", \"dtype_state\": null}, \"components\": \"str\"}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/Uint16.json b/tests/schemas/Uint16.json new file mode 100644 index 0000000..d28feb2 --- /dev/null +++ b/tests/schemas/Uint16.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Uint16\", \"dtype_state\": null}, \"components\": \"uint16\"}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Uint16\", \"dtype_state\": null}, \"components\": \"uint16\"}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/Uint32.json b/tests/schemas/Uint32.json new file mode 100644 index 0000000..4a51ee6 --- /dev/null +++ b/tests/schemas/Uint32.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Uint32\", \"dtype_state\": null}, \"components\": \"uint32\"}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Uint32\", \"dtype_state\": null}, \"components\": \"uint32\"}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/Uint64.json b/tests/schemas/Uint64.json new file mode 100644 index 0000000..9cfd60e --- /dev/null +++ b/tests/schemas/Uint64.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Uint64\", \"dtype_state\": null}, \"components\": \"uint64\"}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Uint64\", \"dtype_state\": null}, \"components\": \"uint64\"}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/schemas/Uint8.json b/tests/schemas/Uint8.json new file mode 100644 index 0000000..8c3e42e --- /dev/null +++ b/tests/schemas/Uint8.json @@ -0,0 +1,3 @@ +{ + "ndonnx": "{\"schemas\": {\"arguments\": {\"a\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Uint8\", \"dtype_state\": null}, \"components\": \"uint8\"}}, \"results\": {\"b\": {\"dtype_info\": {\"defining_library\": \"ndonnx\", \"version\": 1, \"dtype\": \"Uint8\", \"dtype_state\": null}, \"components\": \"uint8\"}}}, \"metadata_version\": \"1\"}" +} diff --git a/tests/test_logic_in_data.py b/tests/test_logic_in_data.py index 1d35915..f88fc7c 100644 --- a/tests/test_logic_in_data.py +++ b/tests/test_logic_in_data.py @@ -7,7 +7,7 @@ import ndonnx._logic_in_data as ndx from ndonnx._logic_in_data import _dtypes as dtypes -from ndonnx._logic_in_data._schema import get_schemas +from ndonnx._logic_in_data._schema import _get_schemas, get_schemas def check_dtype_shape(arr, dtype, shape): @@ -187,17 +187,60 @@ def test_add_pyscalar_timedelta(op): check_dtype_shape(op(arr, scalar), expected_dtype, shape) -def test_build(): - a = ndx.Array(shape=("N",), dtype=ndx.nint64) - b = a[0] +@pytest.mark.parametrize( + "dtype", + [ + ndx.int8, + ndx.int16, + ndx.int32, + ndx.int64, + ndx.uint8, + ndx.uint16, + ndx.uint32, + ndx.uint64, + ndx.float32, + ndx.float64, + ndx.string, + ndx.bool, + ndx.nint8, + ndx.nint16, + ndx.nint32, + ndx.nint64, + ndx.nuint8, + ndx.nuint16, + ndx.nuint32, + ndx.nuint64, + ndx.nfloat32, + ndx.nfloat64, + ndx.nstring, + ndx.nbool, + ], +) +def test_build(dtype): + a = ndx.Array(shape=("N",), dtype=dtype) + b = a[0] # make the build non-trivial mp = ndx.build({"a": a}, {"b": b}) - schemas = get_schemas(mp) + # We must not break backwards compatibility. We test every type we + # support that it keeps producing the same schema. + import json + from pathlib import Path + + fname = Path(__file__).parent / f"schemas/{dtype}.json" + with open(fname, "w+") as f: + json.dump({el.key: el.value for el in mp.metadata_props}, f, indent=4) + f.write("\n") # Avoid pre-commit complaint about missing new lines + + with open(fname) as f: + expected_schemas = _get_schemas(json.load(f)) + candidate_schemas = get_schemas(mp) + + assert expected_schemas == candidate_schemas # test json round trip of schema data - assert schemas.arguments["a"] == a._data.disassemble()[1] - assert schemas.results["b"] == b._data.disassemble()[1] + assert candidate_schemas.arguments["a"] == a._data.disassemble()[1] + assert candidate_schemas.results["b"] == b._data.disassemble()[1] @pytest.mark.parametrize(