From d7c6ca86fd9f553e62c022030abca325a4a5c1b4 Mon Sep 17 00:00:00 2001 From: Jeroen Pinxten Date: Thu, 15 Feb 2024 10:25:26 +0000 Subject: [PATCH] BAC-9152: add customer-results entity to SDK Merge in BAC/icometrix-sdk from feature/customer-results to master Squashed commit of the following: commit 156e8df02234061e10175f27fa3e9b8e0950427f Author: Jeroen Pinxten Date: Thu Feb 15 10:40:47 2024 +0100 BAC-9152: add customer-results entity to SDK --- docs/conf.py | 4 +- docs/developer_guide/index.rst | 3 +- docs/developer_guide/models.rst | 23 ++++++++++ docs/models/customer_result.rst | 6 +++ docs/resources/customer_results.rst | 9 ++++ docs/resources/index.rst | 1 + examples/upload_and_download.py | 35 +++++---------- icometrix_sdk/__init__.py | 6 +++ icometrix_sdk/authentication.py | 6 +++ .../models/customer_report_entity.py | 2 - .../models/customer_result_entity.py | 16 +++++++ icometrix_sdk/resources/customer_reports.py | 26 ++++++++++- icometrix_sdk/resources/customer_results.py | 45 +++++++++++++++++++ icometrix_sdk/utils/paginator.py | 18 ++++---- icometrix_sdk/utils/requests_api_client.py | 8 ++-- 15 files changed, 166 insertions(+), 42 deletions(-) create mode 100644 docs/developer_guide/models.rst create mode 100644 docs/models/customer_result.rst create mode 100644 docs/resources/customer_results.rst create mode 100644 icometrix_sdk/models/customer_result_entity.py create mode 100644 icometrix_sdk/resources/customer_results.py diff --git a/docs/conf.py b/docs/conf.py index ccb4ba4..91b35c3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -9,7 +9,7 @@ # For the full list of built-in configuration values, see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html -root_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) +root_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) sys.path.insert(0, root_path) from icometrix_sdk._version import __version__ @@ -44,4 +44,4 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output html_theme = "furo" -html_static_path = ["_static"] +# html_static_path = ["_static"] diff --git a/docs/developer_guide/index.rst b/docs/developer_guide/index.rst index 8625118..b3216b6 100644 --- a/docs/developer_guide/index.rst +++ b/docs/developer_guide/index.rst @@ -4,4 +4,5 @@ Developer Guide .. toctree:: paginators - session \ No newline at end of file + session + models \ No newline at end of file diff --git a/docs/developer_guide/models.rst b/docs/developer_guide/models.rst new file mode 100644 index 0000000..ae31b17 --- /dev/null +++ b/docs/developer_guide/models.rst @@ -0,0 +1,23 @@ +Models +====== + +Some API operations may return incomplete results that require multiple requests to retrieve the entire +dataset. This process of fetching subsequent pages is known as pagination. Pagination is a crucial aspect when dealing +with large datasets to ensure efficient data retrieval. + +Base model +---------- + +All entities that are returned by the icometrix API extend the :class:`~icometrix_sdk.models.base.BackendEntity` entity. +The BaseEntity always has the following properties: + +- :attr:`~icometrix_sdk.models.base.BackendEntity.id`: The unique identifier (uuid) +- :attr:`~icometrix_sdk.models.base.BackendEntity.update_timestamp`: The timestamp of the last update +- :attr:`~icometrix_sdk.models.base.BackendEntity.creation_timestamp`: The creation timestamp +- :attr:`~icometrix_sdk.models.base.BackendEntity.uri`: The uri to point to the location of the object + +Collections +----------- + +When fetching a collection of models from the API; you will always get a subset of that collection in combination with +some extra meta data. See: :doc:`paginators` \ No newline at end of file diff --git a/docs/models/customer_result.rst b/docs/models/customer_result.rst new file mode 100644 index 0000000..7d6f118 --- /dev/null +++ b/docs/models/customer_result.rst @@ -0,0 +1,6 @@ +Customer Report +=============== + +.. automodule:: icometrix_sdk.models.customer_result_entity + :members: + :undoc-members: diff --git a/docs/resources/customer_results.rst b/docs/resources/customer_results.rst new file mode 100644 index 0000000..65408b0 --- /dev/null +++ b/docs/resources/customer_results.rst @@ -0,0 +1,9 @@ +Customer Results +================ + +.. automodule:: icometrix_sdk.resources.customer_results + + +.. autoclass:: icometrix_sdk.resources.customer_results.CustomerResults + :members: + :show-inheritance: diff --git a/docs/resources/index.rst b/docs/resources/index.rst index 33fb677..859a6ae 100644 --- a/docs/resources/index.rst +++ b/docs/resources/index.rst @@ -7,4 +7,5 @@ Resources projects patients uploads + customer_results customer_reports \ No newline at end of file diff --git a/examples/upload_and_download.py b/examples/upload_and_download.py index 5d42716..52fec3f 100644 --- a/examples/upload_and_download.py +++ b/examples/upload_and_download.py @@ -1,14 +1,12 @@ import logging import os from pathlib import Path -from time import sleep -from typing import List, Dict +from typing import List import pydicom from pydicom.errors import InvalidDicomError from icometrix_sdk import IcometrixApi -from icometrix_sdk.models.customer_report_entity import CustomerReportEntity from icometrix_sdk.models.upload_entity import StartUploadDto PROJECT_ID = "" @@ -20,9 +18,15 @@ logging.basicConfig(level=logging.INFO) -def extract_unique_studies(dir_path: str) -> List[str]: +def extract_unique_studies(folder_path: str) -> List[str]: + """ + Extract all study_instance_uids from DICOMS in a (sub)folder(s) + + :param folder_path: The root path to start searching + :return: + """ study_uids: List[str] = [] - for path, subdirs, files in os.walk(dir_path): + for path, _, files in os.walk(folder_path): for name in files: file_path = os.path.join(path, name) @@ -37,24 +41,6 @@ def extract_unique_studies(dir_path: str) -> List[str]: return study_uids -def wait_for_customer_reports_to_finish(customer_reports: List[CustomerReportEntity]) -> List[CustomerReportEntity]: - finished_customer_reports: Dict[str, CustomerReportEntity] = {} - while len(finished_customer_reports) != len(customer_reports): - sleep(5) - for customer_report in customer_reports: - if customer_report.id in finished_customer_reports: - continue - - report = ico_api.customer_reports.get_one(customer_report.uri) - if report.status != "Finished": - print(f"Waiting for {report.study_instance_uid} to complete: {report.status}") - continue - - print(f"Finished {report.study_instance_uid}") - finished_customer_reports[report.id] = report - return [value for value in finished_customer_reports.values()] - - if __name__ == '__main__': os.environ["API_HOST"] = "https://icobrain-test.icometrix.com" @@ -81,7 +67,8 @@ def wait_for_customer_reports_to_finish(customer_reports: List[CustomerReportEnt customer_reports.append(csr) # Wait for the reports to finish - customer_reports = wait_for_customer_reports_to_finish(customer_reports) + customer_report_uris = [customer_report.uri for customer_report in customer_reports] + customer_reports = ico_api.customer_reports.wait_for_results(customer_report_uris) # Download icobrain report files for customer_report in customer_reports: diff --git a/icometrix_sdk/__init__.py b/icometrix_sdk/__init__.py index ca3f615..375b543 100644 --- a/icometrix_sdk/__init__.py +++ b/icometrix_sdk/__init__.py @@ -6,6 +6,7 @@ from icometrix_sdk.exceptions import IcometrixConfigException from icometrix_sdk.resources.customer_reports import CustomerReports from icometrix_sdk.models.base import PaginatedResponse +from icometrix_sdk.resources.customer_results import CustomerResults from icometrix_sdk.resources.patients import Patients from icometrix_sdk.resources.profile import Profile from icometrix_sdk.resources.projects import Projects @@ -39,6 +40,7 @@ class IcometrixApi: patients: Patients uploads: Uploads customer_reports: CustomerReports + customer_results: CustomerResults _api_client: ApiClient @@ -52,9 +54,13 @@ def __init__(self, api_client: Optional[ApiClient] = None): self.patients = Patients(self._api_client) self.uploads = Uploads(self._api_client) self.customer_reports = CustomerReports(self._api_client) + self.customer_results = CustomerResults(self._api_client) def __enter__(self): return self def __exit__(self, *args, **kwargs): pass + # todo force logout + # if self._api_client.auth: + # self._api_client.auth.disconnect(self._api_client) diff --git a/icometrix_sdk/authentication.py b/icometrix_sdk/authentication.py index 8c858de..5ea1979 100644 --- a/icometrix_sdk/authentication.py +++ b/icometrix_sdk/authentication.py @@ -65,6 +65,12 @@ def get_auth_method() -> Optional[AuthenticationMethod]: Find an authentication method based on the 'AUTH_METHOD' environment variable """ auth_method = os.getenv("AUTH_METHOD") + if auth_method is None: + if os.getenv("TOKEN"): + auth_method = "token" + elif os.getenv("EMAIL") and os.getenv("PASSWORD"): + auth_method = "basic" + if auth_method == "basic": return PasswordAuthentication(os.environ["EMAIL"], os.environ["PASSWORD"]) elif auth_method == "token": diff --git a/icometrix_sdk/models/customer_report_entity.py b/icometrix_sdk/models/customer_report_entity.py index 810c11f..a1f34c4 100644 --- a/icometrix_sdk/models/customer_report_entity.py +++ b/icometrix_sdk/models/customer_report_entity.py @@ -1,7 +1,5 @@ from typing import Optional, Union, List - from pydantic import BaseModel - from icometrix_sdk.models.base import BackendEntity diff --git a/icometrix_sdk/models/customer_result_entity.py b/icometrix_sdk/models/customer_result_entity.py new file mode 100644 index 0000000..76015c3 --- /dev/null +++ b/icometrix_sdk/models/customer_result_entity.py @@ -0,0 +1,16 @@ +from pydantic import BaseModel +from icometrix_sdk.models.base import BackendEntity + + +class CustomerResultPipelineResults(BaseModel): + pipeline_results_id: str + + +class CustomerResultEntity(BackendEntity): + uri: str + job_id: str + study_id: str + patient_id: str + project_id: str + qc_result_id: str + pipeline_results: CustomerResultPipelineResults diff --git a/icometrix_sdk/resources/customer_reports.py b/icometrix_sdk/resources/customer_reports.py index b671849..1c02cd0 100644 --- a/icometrix_sdk/resources/customer_reports.py +++ b/icometrix_sdk/resources/customer_reports.py @@ -1,6 +1,6 @@ import logging from time import sleep -from typing import Optional +from typing import Optional, Dict, List from icometrix_sdk.exceptions import IcometrixDataImportException from icometrix_sdk.models.base import PaginatedResponse @@ -87,3 +87,27 @@ def wait_for_customer_report_for_study(self, project_id: str, if count >= max_count: raise IcometrixDataImportException(f"Failed to find a CustomerReport for {study_instance_uid}") sleep(self.polling_interval) + + def wait_for_results(self, customer_report_uris: List[str]) -> List[CustomerReportEntity]: + """ + Wait until processing has finished and the result files are available on the customer report + + :param customer_report_uris: A list of customer uris + :return: + """ + + finished_customer_reports: Dict[str, CustomerReportEntity] = {} + while len(finished_customer_reports) != len(customer_report_uris): + for customer_report_uri in customer_report_uris: + if customer_report_uri in finished_customer_reports: + continue + + report = self.get_one(customer_report_uri) + if report.status != "Finished": + print(f"Waiting for {report.study_instance_uid} to complete: {report.status}") + continue + + print(f"Finished {report.study_instance_uid}") + finished_customer_reports[report.uri] = report + sleep(self.polling_interval) + return [value for value in finished_customer_reports.values()] diff --git a/icometrix_sdk/resources/customer_results.py b/icometrix_sdk/resources/customer_results.py new file mode 100644 index 0000000..c42ac0d --- /dev/null +++ b/icometrix_sdk/resources/customer_results.py @@ -0,0 +1,45 @@ +import logging + +from icometrix_sdk.models.base import PaginatedResponse +from icometrix_sdk.models.customer_result_entity import CustomerResultEntity +from icometrix_sdk.utils.requests_api_client import ApiClient + +logger = logging.getLogger(__name__) + + +class CustomerResults: + def __init__(self, api: ApiClient): + self._api = api + + def get_all_for_study(self, study_uri: str, **kwargs) -> PaginatedResponse[CustomerResultEntity]: + """ + Get al customer reports for a project + + :param study_uri: The uri of a study + :return: A Paginated response containing customer-results + """ + + page = self._api.get(f"{study_uri}/customer-results", **kwargs) + return PaginatedResponse[CustomerResultEntity](**page) + + def get_all_for_pipeline_result(self, pipeline_result_uri: str, **kwargs) -> \ + PaginatedResponse[CustomerResultEntity]: + """ + Get al customer reports for a pipeline-result + + :param pipeline_result_uri: The uri of a pipeline-result + :return: A Paginated response containing customer-results + """ + + page = self._api.get(f"{pipeline_result_uri}/customer-results", **kwargs) + return PaginatedResponse[CustomerResultEntity](**page) + + def get_one(self, customer_result_uri: str) -> CustomerResultEntity: + """ + Get a single customer-result based on the customer-result uri + + :param customer_result_uri: the uri of the customer-result + :return: A single customer-result or 404 + """ + resp = self._api.get(customer_result_uri) + return CustomerResultEntity(**resp) diff --git a/icometrix_sdk/utils/paginator.py b/icometrix_sdk/utils/paginator.py index 6643c89..1962e64 100644 --- a/icometrix_sdk/utils/paginator.py +++ b/icometrix_sdk/utils/paginator.py @@ -1,9 +1,10 @@ from inspect import signature -from typing import Callable, Generic, TypeVar, Optional +from typing import Callable, Generic, TypeVar, Optional, Iterable, Any, ParamSpec from icometrix_sdk.models.base import PaginatedResponse T = TypeVar("T") +P = ParamSpec("P") def can_paginate(func: Callable): @@ -14,7 +15,7 @@ def can_paginate(func: Callable): return issubclass(sig.return_annotation, PaginatedResponse) -class PageIterator(Generic[T]): +class PageIterator(Generic[T], Iterable): """ An iterable object to iterate over paginated api responses """ @@ -55,7 +56,9 @@ def _fetch_current_page(self) -> PaginatedResponse[T]: return self._func(**self._op_kwargs) -def get_paginator(func: Callable, page_size: Optional[int] = 50, starting_index: Optional[int] = 0, **kwargs): +def get_paginator(func: Callable[..., PaginatedResponse[T]], + page_size: Optional[int] = 50, + starting_index: Optional[int] = 0, **kwargs) -> PageIterator[T]: """ Create paginator object for an operation. @@ -70,12 +73,11 @@ def get_paginator(func: Callable, page_size: Optional[int] = 50, starting_index: The size of the pages :param starting_index: The starting page - :returns: Iterator + :returns: A PageIterator """ if not can_paginate(func): raise ValueError(f"Function '{func.__name__}' can't be paginated") - sig = signature(func) - return iter( - PageIterator[sig.return_annotation](func, op_kwargs=kwargs, page_size=page_size, starting_index=starting_index) - ) + # sig = signature(func) + return PageIterator[T](func, op_kwargs=kwargs, page_size=page_size, + starting_index=starting_index) diff --git a/icometrix_sdk/utils/requests_api_client.py b/icometrix_sdk/utils/requests_api_client.py index e6f988e..a6eff5b 100644 --- a/icometrix_sdk/utils/requests_api_client.py +++ b/icometrix_sdk/utils/requests_api_client.py @@ -33,7 +33,7 @@ class RequestsApiClient(ApiClient): _auth_attempts = 0 _session: Session - _auth: Optional[AuthenticationMethod] + auth: Optional[AuthenticationMethod] def __init__(self, server: str, auth: Optional[AuthenticationMethod] = None): self.base_headers = { @@ -45,7 +45,7 @@ def __init__(self, server: str, auth: Optional[AuthenticationMethod] = None): "Content-Type": "application/json" } self.server = server - self._auth = auth + self.auth = auth if not server: raise IcometrixConfigException("Server is required") @@ -54,10 +54,10 @@ def __init__(self, server: str, auth: Optional[AuthenticationMethod] = None): self._authenticate() def _authenticate(self): - if self._auth: + if self.auth: logger.info("Authenticating") self._auth_attempts += 1 - self._auth.connect(self) + self.auth.connect(self) self._auth_attempts = 0 def _refresh_token(self, resp: Response):