BAC-9366: create docs for data-validation + upload

Merge in BAC/icometrix-sdk from data-validation to master Squashed commit of the following: commit 20adebae92077c0c87781e118a02e43b34024868 Author: Jeroen Pinxten <jeroen.pinxten@icometrix.com> Date: Fri Jun 21 11:04:56 2024 +0200 BAC-9366: improve docs commit e45b5cd8a5fdeac31f987c2a08bb7f97a0fb7024 Author: Jeroen Pinxten <jeroen.pinxten@icometrix.com> Date: Fri Jun 21 10:39:07 2024 +0200 BAC-9366: create docs for data-validation + upload commit 173a4140a82ecc2326c76ac5424baf4e421e1168 Merge: c6a462c f855aba Author: Jeroen Pinxten <jeroen.pinxten@icometrix.com> Date: Wed Jun 19 17:33:16 2024 +0200 Merge branch 'master' into data-validation commit f855aba0bdcf1a39c10c81f2dea4626614041d76 Merge: c54ac15 d977dfd Author: Jeroen Pinxten <jeroen.pinxten@icometrix.com> Date: Wed Jun 19 17:32:43 2024 +0200 Merge branch 'master' of https://stash.icometrix.com/scm/bac/icometrix-sdk commit c6a462c296cbd33a0d12a465735983a932ee659e Author: Jeroen Pinxten <jeroen.pinxten@icometrix.com> Date: Wed Jun 19 17:25:50 2024 +0200 BAC-9366: remove unused import commit 1214b4c19cdefb9ff225c876a4892ff706cea59e Author: Jeroen Pinxten <jeroen.pinxten@icometrix.com> Date: Wed Jun 19 17:18:46 2024 +0200 BAC-9366: data validation docs commit c54ac15fb08952bcc8ea949a5f120bea22173308 Author: Jeroen Pinxten <jeroen.pinxten@icometrix.com> Date: Wed Jun 19 16:59:48 2024 +0200 BAC-9366: data validation examples
icometrix · Jun 21, 2024 · d8e1275 · d8e1275
1 parent b77821c
commit d8e1275
Show file tree

Hide file tree

Showing 7 changed files with 139 additions and 15 deletions.
diff --git a/docs/developer_guide/data_validation.rst b/docs/developer_guide/data_validation.rst
@@ -1,8 +1,73 @@
 Data validation
 ===============
 
-Validating imported data
-------------------------
+When working with DICOM files, ensuring that all necessary files have been correctly uploaded is crucial for
+maintaining data integrity and avoiding potential issues in subsequent processing.
+
+Our SDK provides two methods for validating the completeness of your uploads:
+**pre-import** validation and **post-import** validation. Each method has its own advantages depending on the stage of the process you are in.
+
+Pre-import validation
+---------------------
+
+Pre-import validation allows you to check that all files within a single upload block are accounted for before
+they are imported into the system.
+This method is useful for catching any missing files early in the process, potentially saving time by identifying
+issues before the import step.
+
+How to Perform Pre-Import Validation
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To perform pre-import validation, you can list all the files within a given upload and verify that the expected files are present.
+This has to be done **before completing the upload**. (:meth:`~icometrix_sdk.resources.uploads.Uploads.complete_upload`)
+
+.. code-block:: python
+
+    import logging
+    import os
+
+    from icometrix_sdk import IcometrixApi
+    from icometrix_sdk.models.upload_entity import StartUploadDto
+
+    PROJECT_ID = "<your-project-uuid>"
+
+    # A directory containing a T1 and FLAIR scan, as required for icobrain_ms. For more info, see HCP manual
+    DICOM_DIR_PATH = "..."
+
+    if __name__ == '__main__':
+        os.environ["API_HOST"] = "https://icobrain-{region}.icometrix.com"
+
+        # Initialize the icometrix API
+        ico_api = IcometrixApi()
+
+        # Get the project, to make sure its there (will throw a 404 in case the project is not found)
+        project = ico_api.projects.get_one_by_id(PROJECT_ID)
+
+        # Upload a directory of DICOMs
+        data = StartUploadDto(icobrain_report_type="icobrain_ms")
+        upload = ico_api.uploads.start_upload(PROJECT_ID, data)
+        count = ico_api.uploads.upload_all_files_in_dir(upload.uri, DICOM_DIR_PATH)
+
+        # add validation logic here
+        file_uploads = ico_api.uploads.get_uploaded_files(upload.folder_uri)
+        if len(file_uploads.files) != count:
+            raise Exception("Not all files uploaded")
+
+        # Once all files have been uploaded, signal that they are all there and start the import/processing
+        upload = ico_api.uploads.complete_upload(upload.uri)
+
+
+Post-import validation
+----------------------
+
+Post-import validation occurs after the DICOM files have been imported into the system. This method ensures that all
+hierarchical :doc:`models` (Patients, Studies, Series, and individual DICOM files) have been correctly imported.
+
+How to Perform Post-Import Validation
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To perform post-import validation, you can query the system for the imported entities and verify that all expected
+Patients, Studies, Series, and DICOM files are present.
 
 .. code-block:: python
 

diff --git a/docs/developer_guide/index.rst b/docs/developer_guide/index.rst
@@ -6,5 +6,6 @@ Developer Guide
    paginators
    session
    models
+   upload
    anonymization
    data_validation
diff --git a/docs/developer_guide/model_relations.mmd b/docs/developer_guide/model_relations.mmd
@@ -4,7 +4,7 @@ erDiagram
    PATIENTS ||--|{ STUDIES : contains
    STUDIES ||--|{ SERIES : contains
    SERIES ||--|{ DICOMS : contains
-   STUDIES ||--|{ PIPELINE-RESULT : has
+   STUDIES ||--|{ PIPELINE-RESULTS : has
    PIPELINE-RESULTS ||--|{ CUSTOMER-RESULTS : has
    PATIENTS ||--|{ PIPELINE-LONG-RESULTS : has
    PIPELINE-LONG-RESULTS ||--|{ CUSTOMER-LONG-RESULTS : has

diff --git a/docs/developer_guide/upload.rst b/docs/developer_guide/upload.rst
@@ -0,0 +1,59 @@
+How to Upload
+=============
+
+Uploading DICOM files using our Python SDK is a straightforward process that involves three main steps:
+
+1. **starting the upload**
+
+2. **uploading the files**
+
+3. **completing the upload**.
+
+You can choose to upload files one by one or use a higher-level function to upload all files in a directory.
+Below, we outline these steps and provide example code to help you get started.
+
+Upload All Files in a Directory
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To simplify the process, you can use the :meth:`~icometrix_sdk.resources.uploads.Uploads.upload_all_files_in_dir` method, which will walk through all files and
+subdirectories within a specified directory and upload each file.
+
+.. code-block:: python
+
+    import os
+    from icometrix_sdk import IcometrixApi
+    from icometrix_sdk.models.upload_entity import StartUploadDto
+
+    PROJECT_ID = "uuid"
+    DICOM_DIR_PATH = "<dir_path>"
+
+    if __name__ == '__main__':
+        os.environ["API_HOST"] = "https://icobrain-{region}.icometrix.com"
+
+        # Initialize the icometrix API
+        ico_api = IcometrixApi()
+
+        # Get the project, to make sure it's there (will throw a 404 in case the project is not found)
+        project = ico_api.projects.get_one_by_id(PROJECT_ID)
+
+        # Start an upload
+        data = StartUploadDto(icobrain_report_type="icobrain_ms")
+        upload = ico_api.uploads.start_upload(PROJECT_ID, data)
+
+        # Will walk through all files/subdirectories and upload all files
+        ico_api.uploads.upload_all_files_in_dir(upload.uri, DICOM_DIR_PATH)
+
+        # Once all files have been uploaded, signal that they are all there and start the import/processing
+        upload = ico_api.uploads.complete_upload(upload.uri)
+
+
+Upload Files Individually
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you prefer to upload files one by one, you can use the upload_file method.
+This method requires the upload URI and the file path for each file you wish to upload.
+
+.. code-block:: python
+
+    ico_api.uploads.upload_dicom_path(upload.uri, dicom_path)
+
diff --git a/docs/quickstart.rst b/docs/quickstart.rst
@@ -29,7 +29,7 @@ Next thing is to set the correct server region (icobrain-eu.icometrix.com, icobr
 
     from icometrix_sdk import IcometrixApi
 
-    SERVER = "https://icobrain-test.icometrix.com"
+    SERVER = "https://icobrain-{region}.icometrix.com"
 
 By default the icometrix_sdk will try to auto detect the authentication method. But we can
 also force a method e.g. password authentication. To do this import the PasswordAuthentication
@@ -39,7 +39,7 @@ method and pass it as a parameter to the IcometrixApi.
 
     from icometrix_sdk import IcometrixApi
 
-    SERVER = "https://icobrain-test.icometrix.com"
+    SERVER = "https://icobrain-{region}.icometrix.com"
 
     auth = PasswordAuthentication("example@company.com", os.environ["PASSWORD"])
     client = RequestsApiClient(SERVER, auth)
@@ -52,7 +52,7 @@ You can use :meth:`~icometrix_sdk.resources.profile.Profile.who_am_i` function t
 
     from icometrix_sdk import IcometrixApi
 
-    SERVER = "https://icobrain-test.icometrix.com"
+    SERVER = "https://icobrain-{region}.icometrix.com"
 
     auth = PasswordAuthentication("example@company.com", os.environ["PASSWORD"])
     client = RequestsApiClient(SERVER, auth)

diff --git a/examples/upload_directory.py b/examples/upload_directory.py
@@ -7,22 +7,20 @@
 DICOM_DIR_PATH = "<dir_path>"
 
 if __name__ == '__main__':
-    os.environ["API_HOST"] = "https://icobrain-test.icometrix.com"
+    os.environ["API_HOST"] = "https://icobrain-{region}.icometrix.com"
 
     # Initialize the icometrix API
     ico_api = IcometrixApi()
 
     # Get the project, to make sure its there (will throw a 404 in case the project is not found)
     project = ico_api.projects.get_one_by_id(PROJECT_ID)
 
-    # Upload a directory of DICOMs
+    # Start an upload
     data = StartUploadDto(icobrain_report_type="icobrain_ms")
     upload = ico_api.uploads.start_upload(PROJECT_ID, data)
-    ico_api.uploads.upload_dicom_dir(upload.uri, DICOM_DIR_PATH)
 
-    # Wait for data to be imported
-    upload = ico_api.uploads.wait_for_data_import(upload.folder_uri)
+    # Will walk through all files/subdirectories and upload all files
+    ico_api.uploads.upload_all_files_in_dir(upload.uri, DICOM_DIR_PATH)
 
-    # Get imported studies
-    for uploaded_study in ico_api.uploads.get_studies_for_upload(upload_folder_uri=upload.folder_uri):
-        print(uploaded_study.study_id)
+    # Once all files have been uploaded, signal that they are all there and start the import/processing
+    upload = ico_api.uploads.complete_upload(upload.uri)
diff --git a/icometrix_sdk/models/customer_result_entity.py b/icometrix_sdk/models/customer_result_entity.py
@@ -1,10 +1,11 @@
 from pydantic import BaseModel
+from typing_extensions import Optional
 
 from icometrix_sdk.models.base import BackendEntity
 
 
 class CustomerResultPipelineResults(BaseModel):
-    pipeline_results_id: str
+    pipeline_results_id: Optional[str] = None
 
 
 class CustomerResultEntity(BackendEntity):