ENH: Create unit tests for each python example pipeline (#793)

Signed-off-by: Michael Jackson <mike.jackson@bluequartz.net>

.github/workflows/linux.yml

@@ -36,14 +36,23 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: true
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install sphinx myst-parser sphinx-markdown-tables sphinx_rtd_theme numpy
       - name: Add C++ Problem Matcher
         uses: ammaraskar/gcc-problem-matcher@0.2.0
-      - name: Install Dependencies
+      - name: Install Dependencies - 2
         run: |
           sudo apt-get -y install ninja-build
       - name: Install Sphinx
         run: |
-          sudo pip install sphinx myst-parser sphinx-markdown-tables sphinx_rtd_theme numpy
+          sudo pip3 install sphinx myst-parser sphinx-markdown-tables sphinx_rtd_theme numpy
       - name: Setup NuGet Credentials
         shell: bash
         run: |

.github/workflows/macos.yml

@@ -24,10 +24,19 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
         with:
-          submodules: true 
+          submodules: true
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install sphinx myst-parser sphinx-markdown-tables sphinx_rtd_theme numpy
       - name: Add C++ Problem Matcher
         uses: ammaraskar/gcc-problem-matcher@0.2.0
-      - name: Install Dependencies
+      - name: Install Dependencies - 2
         run: |
           brew install ninja
       - name: Install Sphinx

CMakePresets.json

@@ -33,6 +33,10 @@
           "type": "BOOL",
           "value": "ON"
         },
+        "SIMPLNX_ENABLE_PYTHON_TESTS": {
+          "type": "BOOL",
+          "value": "ON"
+        },
         "SIMPLNX_EMBED_PYTHON": {
           "type": "BOOL",
           "value": "OFF"
@@ -121,6 +125,14 @@
         "VCPKG_HOST_TRIPLET": {
           "type": "STRING",
           "value": "x64-osx-v11"
+        },
+        "Python3_EXECUTABLE": {
+          "type": "PATH",
+          "value": "/Users/runner/hostedtoolcache/Python/3.10.13/x64/bin/python3.10"
+        },
+        "SIMPLNX_PY_DISABLE_HIDDEN_VISIBILITY": {
+          "type": "BOOL",
+          "value": "ON"
         }
       }
     },
@@ -138,6 +150,14 @@
         "VCPKG_HOST_TRIPLET": {
           "type": "STRING",
           "value": "arm64-osx-dynamic"
+        },
+        "Python3_EXECUTABLE": {
+          "type": "PATH",
+          "value": "/Users/runner/hostedtoolcache/Python/3.10.13/x64/bin/python3.10"
+        },
+        "SIMPLNX_PY_DISABLE_HIDDEN_VISIBILITY": {
+          "type": "BOOL",
+          "value": "ON"
         }
       }
     },
@@ -155,6 +175,10 @@
         "VCPKG_HOST_TRIPLET": {
           "type": "STRING",
           "value": "x64-linux-dynamic"
+        },
+        "Python3_EXECUTABLE": {
+          "type": "PATH",
+          "value": "/opt/hostedtoolcache/Python/3.10.13/x64/bin/python3.10"
         }
       }
     },
@@ -278,6 +302,10 @@
         "SIMPLNX_PY_DISABLE_HIDDEN_VISIBILITY": {
           "type": "BOOL",
           "value": "ON"
+        },
+        "Python3_EXECUTABLE": {
+          "type": "PATH",
+          "value": "/opt/local/anaconda3/envs/nx-build/bin/python"
         }
       }
     },

cmake/Utility.cmake

@@ -460,3 +460,86 @@ function(cmpBuildDateRevisionString)
   endif()
 
 endfunction()
+
+
+#-------------------------------------------------------------------------------
+# @Brief function AddPythonTest
+# @ NAME
+# @ FILE
+# @ PYTHONPATH
+#-------------------------------------------------------------------------------
+function(AddPythonTest)
+  set(options )
+  set(oneValueArgs NAME FILE)
+  set(multiValueArgs PYTHONPATH)
+  cmake_parse_arguments(ARGS "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+  message(STATUS "ARGS_FILE:${ARGS_FILE}")
+  if(COMPLEX_BUILD_PYTHON)
+    if(WIN32)
+      add_test(NAME ${ARGS_NAME}
+        COMMAND ${complex_SOURCE_DIR}/wrapping/python/testing/anaconda_test.bat
+      )
+
+      set_property(TEST ${ARGS_NAME}
+        PROPERTY
+          ENVIRONMENT
+            "PYTHON_TEST_FILE=${ARGS_FILE}"
+            "Python3_EXECUTABLE=${Python3_EXECUTABLE}"
+    )
+    else()
+      add_test(NAME ${ARGS_NAME}
+        COMMAND ${complex_SOURCE_DIR}/wrapping/python/testing/anaconda_test.sh
+      )
+      set_property(TEST ${ARGS_NAME}
+        PROPERTY
+          ENVIRONMENT
+            "PYTHON_TEST_FILE=${ARGS_FILE}"
+            "Python3_EXECUTABLE=${Python3_EXECUTABLE}"
+      )    
+    endif()
+  else()
+    add_test(NAME ${ARGS_NAME}
+      COMMAND ${Python3_EXECUTABLE} ${ARGS_FILE}
+    )
+  endif()
+
+  if(WIN32)
+    string(REPLACE ";" "\\;" ARGS_PYTHONPATH "${ARGS_PYTHONPATH}")
+  else()
+    string(REPLACE ";" ":" ARGS_PYTHONPATH "${ARGS_PYTHONPATH}")
+  endif()
+
+  set_property(TEST ${ARGS_NAME}
+    APPEND
+    PROPERTY
+      ENVIRONMENT
+      "PYTHONPATH=${ARGS_PYTHONPATH}"
+  )
+endfunction()
+
+#-------------------------------------------------------------------------------
+# @Brief function CreatePythonTests
+# @ PREFIX
+# @ INPUT_DIR
+# @ TEST_NAMES
+#-------------------------------------------------------------------------------
+function(CreatePythonTests)
+  set(options)
+  set(oneValueArgs PREFIX INPUT_DIR)
+  set(multiValueArgs TEST_NAMES)
+  cmake_parse_arguments(ARGS "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+
+  set(TESTS_PYTHONPATH
+    "$<TARGET_FILE_DIR:complex>"
+  )
+
+  foreach(test ${ARGS_TEST_NAMES})
+    string(REPLACE "/" "_" test_name ${test})
+    set(PY_TEST_NAME ${ARGS_PREFIX}_${test_name})
+
+    AddPythonTest(NAME ${PY_TEST_NAME}
+      FILE ${ARGS_INPUT_DIR}/${test}.py
+      PYTHONPATH ${TESTS_PYTHONPATH}
+    )
+  endforeach()
+endfunction()

src/Plugins/SimplnxCore/wrapping/python/simplnxpy.cpp

@@ -544,8 +544,11 @@ PYBIND11_MODULE(simplnx, mod)
       std::vector<DataPath> outputPaths;
       for(const auto* object : self.getTopLevelData())
       {
-        auto topLevelPath = DataPath::FromString(object->getDataPaths()[0].getTargetName()).value();
-        outputPaths.push_back(topLevelPath);
+        if(object != nullptr)
+        {
+          auto topLevelPath = DataPath::FromString(object->getDataPaths()[0].getTargetName()).value();
+          outputPaths.push_back(topLevelPath);
+        }
       }
       return outputPaths;
     }

src/simplnx/Utilities/DataGroupUtilities.cpp

@@ -186,7 +186,6 @@ std::optional<std::vector<DataPath>> GetAllChildDataPaths(const DataStructure& d
     {
       bool ignore = false;
       DataPath childPath = parentGroup.createChildPath(childName);
-      const DataObject* dataObject = dataStructure.getData(childPath);
       for(const auto& ignoredPath : ignoredDataPaths)
       {
         if(childPath == ignoredPath)
@@ -195,7 +194,8 @@ std::optional<std::vector<DataPath>> GetAllChildDataPaths(const DataStructure& d
           break;
         }
       }
-      if(!ignore && (dataObjectType == DataObject::Type::DataObject || dataObject->getDataObjectType() == dataObjectType))
+      const DataObject* dataObject = dataStructure.getData(childPath);
+      if(dataObject != nullptr && !ignore && (dataObjectType == DataObject::Type::DataObject || dataObject->getDataObjectType() == dataObjectType))
       {
         childDataObjects.push_back(childPath);
       }
@@ -211,7 +211,7 @@ std::optional<std::vector<DataPath>> GetAllChildDataPaths(const DataStructure& d
 {
   std::vector<DataPath> childDataObjects;
   const DataObject* dataObject1 = dataStructure.getData(parent);
-  if(dataObject1->getDataObjectType() == DataObject::Type::DataArray || dataObject1->getDataObjectType() == DataObject::Type::DynamicListArray ||
+  if(dataObject1 == nullptr || dataObject1->getDataObjectType() == DataObject::Type::DataArray || dataObject1->getDataObjectType() == DataObject::Type::DynamicListArray ||
      dataObject1->getDataObjectType() == DataObject::Type::NeighborList || dataObject1->getDataObjectType() == DataObject::Type::ScalarData ||
      dataObject1->getDataObjectType() == DataObject::Type::StringArray)
   {

wrapping/python/CMakeLists.txt

@@ -109,8 +109,8 @@ function(simplnx_create_pyi)
   set(PYI_TARGET_NAME ${ARGS_PYTHON_MODULE_NAME}CreateStubFile)
 
   add_custom_target(${PYI_TARGET_NAME} ALL
-    COMMAND ${MYPY_STUBGEN_EXE} -m ${ARGS_PYTHON_MODULE_NAME} -o $<TARGET_FILE_DIR:simplnx>
-    COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/fix_pyi_file.py --input_file "$<TARGET_FILE_DIR:simplnx>/${ARGS_PYTHON_MODULE_NAME}.pyi"
+    COMMAND "${MYPY_STUBGEN_EXE}" -m "${ARGS_PYTHON_MODULE_NAME}" -o "$<TARGET_FILE_DIR:simplnx>"
+    COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/fix_pyi_file.py" --input_file "$<TARGET_FILE_DIR:simplnx>/${ARGS_PYTHON_MODULE_NAME}.pyi"
     COMMENT "${ARGS_PYTHON_MODULE_NAME}: Generating .pyi files"
     WORKING_DIRECTORY $<TARGET_FILE_DIR:simplnx>
   )
@@ -278,3 +278,46 @@ option(SIMPLNX_ENABLE_SPHINX_DOCS "Enables the Sphinx based documentation genera
 if(SIMPLNX_ENABLE_SPHINX_DOCS)
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/docs ${simplnx_BINARY_DIR}/sphinx_docs)
 endif()
+
+#------------------------------------------------------------------------------
+# Create the Python Unit tests
+#------------------------------------------------------------------------------
+option(COMPLEX_ENABLE_PYTHON_TESTS "Enables python based unit tests" OFF)
+if(COMPLEX_ENABLE_PYTHON_TESTS)
+
+    file(READ ${complex_SOURCE_DIR}/wrapping/python/complex_test_dirs.in.py COMPLEX_TEST_DIRS_FILE)
+
+    string(REPLACE "\${CMAKE_LIBRARY_OUTPUT_DIRECTORY}" $<TARGET_FILE_DIR:complex> COMPLEX_TEST_DIRS_FILE ${COMPLEX_TEST_DIRS_FILE})
+    string(REPLACE "\${complex_BINARY_DIR}" ${complex_BINARY_DIR} COMPLEX_TEST_DIRS_FILE ${COMPLEX_TEST_DIRS_FILE})
+    string(REPLACE "\${DREAM3D_DATA_DIR}" ${DREAM3D_DATA_DIR} COMPLEX_TEST_DIRS_FILE ${COMPLEX_TEST_DIRS_FILE})
+    string(REPLACE "\${complex_SOURCE_DIR}" ${complex_SOURCE_DIR} COMPLEX_TEST_DIRS_FILE ${COMPLEX_TEST_DIRS_FILE})
+
+    file(GENERATE
+      OUTPUT "$<TARGET_FILE_DIR:complex>/complex_test_dirs.py"
+      CONTENT ${COMPLEX_TEST_DIRS_FILE}
+    )
+
+    set(PYTHON_TEST_INPUT_DIR "${complex_SOURCE_DIR}/wrapping/python/examples")
+
+    set(COMPLEX_PYTHON_TESTS
+      "angle_conversion" 
+      "basic_arrays" 
+      "basic_ebsd_ipf"
+      "basic_numpy" 
+      "create_ensemble_info" 
+      "generated_file_list" 
+      "geometry_examples" 
+      "import_d3d"      # Dependent on 'basic_ebsd_ipf' running first
+      "import_hdf5"     # Dependent on 'basic_ebsd_ipf' running first
+      "output_file" 
+      "pipeline" 
+      "read_csv_file"
+    #  "read_esprit_data"
+    )
+
+    CreatePythonTests(PREFIX "PY_COMPLEX"
+      INPUT_DIR ${PYTHON_TEST_INPUT_DIR}
+      TEST_NAMES ${COMPLEX_PYTHON_TESTS}
+    )
+
+endif()

wrapping/python/CxPybind/CxPybind/CxPybind.hpp

@@ -397,7 +397,18 @@ auto BindFilter(py::handle scope, const Internals& internals)
 
     std::string executeSig = MakePythonSignature<FilterT>("execute", internals);
     std::string executeDocString = fmt::format("{}\n\nExecutes the filter\n", executeSig);
-
+    filter.def_static("human_name", [&internals]() {
+      FilterT filter;
+      return filter.humanName();
+    });
+    filter.def_static("name", [&internals]() {
+      FilterT filter;
+      return filter.name();
+    });
+    filter.def_static("uuid", [&internals]() {
+      FilterT filter;
+      return filter.uuid();
+    });
     filter.def_static(
         "execute",
         [&internals](DataStructure& dataStructure, const py::kwargs& kwargs) {

wrapping/python/complex_test_dirs.in.py

@@ -0,0 +1,50 @@
+''' module to get build specific directories
+    This file is generated during the cmake configure phase. This file does *NOT*
+    however use the standard `cmake_configure_file()` command but instead uses
+    a custom generation cmake code. If you add new `${}` style variables, you will
+    need to update those custom CMake codes.
+
+ '''
+
+
+import complex as cx
+
+def check_filter_result(filter: cx.IFilter, result: cx.IFilter.ExecuteResult):
+  if len(result.warnings) != 0:
+    print(f'{filter.name()} ::  Warnings: {result.warnings}')
+
+  has_errors = len(result.errors) != 0 
+  if has_errors:
+    print(f'{filter.name()} :: Errors: {result.errors}')
+    raise RuntimeError(result)
+
+  print(f"{filter.name()} :: No errors running the filter")
+
+
+def GetBuildDirectory():
+  return '${CMAKE_LIBRARY_OUTPUT_DIRECTORY}'
+
+def GetTestDirectory():
+  return '${complex_BINARY_DIR}/Testing'
+
+def GetTestTempDirectory():
+  return '${complex_BINARY_DIR}/Testing/Temporary'
+
+def GetDataDirectory():
+  return '${DREAM3D_DATA_DIR}'
+
+def GetComplexPythonSourceDir():
+    return '${complex_SOURCE_DIR}/wrapping/python'
+
+def GetComplexSourceDir():
+    return '${complex_SOURCE_DIR}'
+
+def print_all_paths():
+    print(f'#### Important Filesystem Paths ####')
+    print(f'  GetBuildDirectory:         {GetBuildDirectory()}')
+    print(f'  GetTestDirectory:          {GetTestDirectory()}')
+    print(f'  GetTestTempDirectory:      {GetTestTempDirectory()}')
+    print(f'  GetDataDirectory:          {GetDataDirectory()}')
+    print(f'  GetComplexPythonSourceDir: {GetComplexPythonSourceDir()}')
+    print(f'  GetComplexSourceDir:       {GetComplexSourceDir()}')
+    print('#######################################')