From 65e9c76722a2733707e8dd670a9eccf66966aef7 Mon Sep 17 00:00:00 2001 From: Michael Jackson Date: Wed, 17 Jul 2024 18:13:47 -0400 Subject: [PATCH] Use ProxMox for builds Signed-off-by: Michael Jackson --- .azure/pipelines/azure-pipelines.yml | 64 +++++++++++++++------ .github/workflows/linux.yml | 76 ------------------------ .github/workflows/macos.yml | 78 ------------------------- .github/workflows/windows.yml | 61 -------------------- CMakePresets.json | 86 ++++++++++++++++++++++++++++ 5 files changed, 133 insertions(+), 232 deletions(-) delete mode 100644 .github/workflows/linux.yml delete mode 100644 .github/workflows/macos.yml delete mode 100644 .github/workflows/windows.yml diff --git a/.azure/pipelines/azure-pipelines.yml b/.azure/pipelines/azure-pipelines.yml index f96c9ad2f0..ab93c538c5 100644 --- a/.azure/pipelines/azure-pipelines.yml +++ b/.azure/pipelines/azure-pipelines.yml @@ -14,6 +14,14 @@ resources: type: github endpoint: BlueQuartzSoftware name: BlueQuartzSoftware/simplnx + - repository: FileStore + type: github + endpoint: BlueQuartzSoftware + name: BlueQuartzSoftware/FileStore + - repository: SimplnxReview + type: github + endpoint: BlueQuartzSoftware + name: BlueQuartzSoftware/SimplnxReview trigger: - develop @@ -21,26 +29,26 @@ trigger: jobs: - job: strategy: - matrix: - macOS: - imageName: Darwin - preset_name: ci-macos-x64 - home_dir: /Users/buildbot - Windows: + matrix-OoC: + # macOS: + # imageName: Darwin + # preset_name: ci-macos-x64 + # home_dir: /Users/buildbot + Windows-OoC: imageName: Windows_NT - preset_name: ci-windows-v143 + preset_name: ci-windows-v143-ooc home_dir: C:/Users/buildbot - Linux: + Linux-OoC: imageName: Linux - preset_name: ci-linux-x64 + preset_name: ci-linux-x64-ooc home_dir: /home/buildbot - python_dir: /opt/hostedtoolcache/Python/3.10.13/x64/bin + python_dir: /opt/local/anaconda3/envs/dream3d/bin pool: name: BlueQuartz Self Hosted demands: - Agent.OS -equals $(imageName) - - BQ.SIMPLNX + - BQ.PROXMOX workspace: clean: all @@ -48,7 +56,9 @@ jobs: timeoutInMinutes: 120 variables: - simplnx_source_dir: $(Build.Repository.LocalPath) + simplnx_source_dir: $(Build.Repository.LocalPath)/simplnx + filestore_source_dir: $(Build.Repository.LocalPath)/FileStore + simplnx_review_source_dir: $(Build.Repository.LocalPath)/SimplnxReview dream3d_data_dir: $(Agent.WorkFolder)/DREAM3D_Data model_type: Experimental build_type: Release @@ -58,6 +68,10 @@ jobs: steps: - checkout: self submodules: true + - checkout: FileStore + submodules: true + - checkout: SimplnxReview + submodules: true - powershell: | try { @@ -81,13 +95,29 @@ jobs: # This also allows CDash to get the real git commit that we are on for DREAM3DNX #================================================================================================== - powershell: | - cd $(Build.Repository.LocalPath) + cd $(Build.Repository.LocalPath)/simplnx git config core.sshCommand "ssh -i $(home_dir)/.ssh/id_rsa" git remote rename origin azure git remote add origin ssh://git@github.com/$(simplnx_origin_name)/simplnx git fetch origin displayName: Update Simplnx Repo - + + - powershell: | + cd $(Build.Repository.LocalPath)/FileStore + git config core.sshCommand "ssh -i $(home_dir)/.ssh/id_rsa" + git remote rename origin azure + git remote add origin ssh://git@github.com/$(simplnx_origin_name)/FileStore + git fetch origin + displayName: Update FileStore Repo + + - powershell: | + cd $(Build.Repository.LocalPath)/SimplnxReview + git config core.sshCommand "ssh -i $(home_dir)/.ssh/id_rsa" + git remote rename origin azure + git remote add origin ssh://git@github.com/$(simplnx_origin_name)/SimplnxReview + git fetch origin + displayName: Update SimplnxReview Repo + #================================================================================================== # CDash Section: # In this section we are going to call `ctest` 3 times in order to separate out the output from @@ -99,17 +129,17 @@ jobs: $UpdatedPath = '$(python_dir)' + [IO.Path]::PathSeparator + [System.Environment]::GetEnvironmentVariable('PATH', [System.EnvironmentVariableTarget]::Process) [System.Environment]::SetEnvironmentVariable('PATH', $UpdatedPath, [System.EnvironmentVariableTarget]::Process) cd $(Build.BinariesDirectory) - ctest -VV -DCTEST_SOURCE_DIR=$(simplnx_source_dir) -DCTEST_BINARY_DIRECTORY=$(Build.BinariesDirectory) -Ddashboard_model=Experimental -DCTEST_DASHBOARD_ROOT=$(Build.Repository.LocalPath) -DCMAKE_PRESET_NAME="$(preset_name)" -DCTEST_CONFIGURATION_TYPE=Release -S $(Build.Repository.LocalPath)/.azure/azure_ci_configure.cmake + ctest -VV -DCTEST_SOURCE_DIR=$(simplnx_source_dir) -DCTEST_BINARY_DIRECTORY=$(Build.BinariesDirectory) -Ddashboard_model=Experimental -DCTEST_DASHBOARD_ROOT=$(Build.Repository.LocalPath) -DCMAKE_PRESET_NAME="$(preset_name)" -DCTEST_CONFIGURATION_TYPE=Release -S $(Build.Repository.LocalPath)/simplnx/.azure/azure_ci_configure.cmake displayName: CDash Update/Configure - powershell: | cd $(Build.BinariesDirectory) - ctest -VV -DCTEST_SOURCE_DIR=$(simplnx_source_dir) -DCTEST_BINARY_DIRECTORY=$(Build.BinariesDirectory) -Ddashboard_model=Experimental -DCTEST_DASHBOARD_ROOT=$(Build.Repository.LocalPath) -DCMAKE_PRESET_NAME="$(preset_name)" -DCTEST_CONFIGURATION_TYPE=Release -S $(Build.Repository.LocalPath)/.azure/azure_ci_build.cmake + ctest -VV -DCTEST_SOURCE_DIR=$(simplnx_source_dir) -DCTEST_BINARY_DIRECTORY=$(Build.BinariesDirectory) -Ddashboard_model=Experimental -DCTEST_DASHBOARD_ROOT=$(Build.Repository.LocalPath) -DCMAKE_PRESET_NAME="$(preset_name)" -DCTEST_CONFIGURATION_TYPE=Release -S $(Build.Repository.LocalPath)/simplnx/.azure/azure_ci_build.cmake displayName: CDash Build - powershell: | cd $(Build.BinariesDirectory) - ctest -VV -DCTEST_SOURCE_DIR=$(simplnx_source_dir) -DCTEST_BINARY_DIRECTORY=$(Build.BinariesDirectory) -Ddashboard_model=Experimental -DCTEST_DASHBOARD_ROOT=$(Build.Repository.LocalPath) -DCMAKE_PRESET_NAME="$(preset_name)" -DCTEST_CONFIGURATION_TYPE=Release -S $(Build.Repository.LocalPath)/.azure/azure_ci_test.cmake + ctest -VV -DCTEST_SOURCE_DIR=$(simplnx_source_dir) -DCTEST_BINARY_DIRECTORY=$(Build.BinariesDirectory) -Ddashboard_model=Experimental -DCTEST_DASHBOARD_ROOT=$(Build.Repository.LocalPath) -DCMAKE_PRESET_NAME="$(preset_name)" -DCTEST_CONFIGURATION_TYPE=Release -S $(Build.Repository.LocalPath)/simplnx/.azure/azure_ci_test.cmake displayName: CDash Test #================================================================================================== diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml deleted file mode 100644 index f9ff17d296..0000000000 --- a/.github/workflows/linux.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: linux - -on: - pull_request: - branches: - - develop - - master - push: - branches: - - develop - - master - -jobs: - build: - env: - VCPKG_BINARY_SOURCES: 'clear;nuget,GitHub,readwrite' - strategy: - fail-fast: false - matrix: - os: - - ubuntu-20.04 - cxx: - - g++-10 - - clang++-10 - include: - - cxx: g++-10 - cc: gcc-10 - - cxx: clang++-10 - cc: clang-10 - runs-on: ${{matrix.os}} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install sphinx myst-parser sphinx-markdown-tables sphinx_rtd_theme numpy - - name: Add C++ Problem Matcher - uses: ammaraskar/gcc-problem-matcher@0.2.0 - - name: Install Dependencies - 2 - run: | - sudo apt-get -y install ninja-build - - name: Install Sphinx - run: | - sudo pip3 install sphinx myst-parser sphinx-markdown-tables sphinx_rtd_theme numpy - - name: Setup NuGet Credentials - shell: bash - run: | - mono `vcpkg fetch nuget | tail -n 1` \ - sources add \ - -source "https://nuget.pkg.github.com/BlueQuartzSoftware/index.json" \ - -storepasswordincleartext \ - -name "GitHub" \ - -username "BlueQuartzSoftware" \ - -password "${{secrets.GITHUB_TOKEN}}" - mono `vcpkg fetch nuget | tail -n 1` \ - setapikey "${{secrets.GITHUB_TOKEN}}" \ - -source "https://nuget.pkg.github.com/BlueQuartzSoftware/index.json" - - name: Configure - env: - CC: ${{matrix.cc}} - CXX: ${{matrix.cxx}} - run: | - cmake --preset ci-linux-x64 ${{github.workspace}} - - name: Build - run: | - cmake --build --preset ci-linux-x64 - - name: Test - run: | - ctest --preset ci-linux-x64 diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml deleted file mode 100644 index e6f7e6e44c..0000000000 --- a/.github/workflows/macos.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: macos - -on: - pull_request: - branches: - - develop - - master - push: - branches: - - develop - - master - -jobs: - build: - env: - VCPKG_BINARY_SOURCES: 'clear;nuget,GitHub,readwrite' - strategy: - fail-fast: false - matrix: - os: - - macos-12 - - macos-14 - include: - - os: macos-12 - preset: ci-macos-x64 - - os: macos-14 - preset: ci-macos-arm64 - runs-on: ${{matrix.os}} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - - name: Checkout vcpkg - run: | - git clone https://www.github.com/microsoft/vcpkg && cd vcpkg && ./bootstrap-vcpkg.sh - VCPKG_INSTALLATION_ROOT=${{github.workspace}}/vcpkg - echo "$VCPKG_INSTALLATION_ROOT" >> $GITHUB_PATH - echo "VCPKG_INSTALLATION_ROOT=$VCPKG_INSTALLATION_ROOT" >> "$GITHUB_ENV" - if: matrix.os == 'macos-14' - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install sphinx myst-parser sphinx-markdown-tables sphinx_rtd_theme numpy - - name: Add C++ Problem Matcher - uses: ammaraskar/gcc-problem-matcher@0.2.0 - - name: Install Dependencies - 2 - run: | - brew install ninja - - name: Install Sphinx - run: | - sudo pip3 install sphinx myst-parser sphinx-markdown-tables sphinx_rtd_theme numpy - - name: Setup NuGet Credentials - shell: bash - run: | - mono `vcpkg fetch nuget | tail -n 1` \ - sources add \ - -source "https://nuget.pkg.github.com/BlueQuartzSoftware/index.json" \ - -storepasswordincleartext \ - -name "GitHub" \ - -username "BlueQuartzSoftware" \ - -password "${{secrets.GITHUB_TOKEN}}" - mono `vcpkg fetch nuget | tail -n 1` \ - setapikey "${{secrets.GITHUB_TOKEN}}" \ - -source "https://nuget.pkg.github.com/BlueQuartzSoftware/index.json" - - name: Configure - run: | - cmake --preset ${{matrix.preset}} ${{github.workspace}} - - name: Build - run: | - cmake --build --preset ${{matrix.preset}} - - name: Test - run: | - ctest --preset ${{matrix.preset}} diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml deleted file mode 100644 index ca2981560a..0000000000 --- a/.github/workflows/windows.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: windows - -on: - pull_request: - branches: - - develop - - master - push: - branches: - - develop - - master - -jobs: - build: - env: - VCPKG_BINARY_SOURCES: 'clear;nuget,GitHub,readwrite' - strategy: - fail-fast: false - matrix: - os: - - windows-2022 - toolset: - - v142 - - v143 - runs-on: ${{matrix.os}} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - - name: Add C++ Problem Matcher - uses: ammaraskar/msvc-problem-matcher@0.2.0 - - name: Setup Build Environment - uses: ilammy/msvc-dev-cmd@v1.12.1 - with: - vsversion: 2022 - - name: Setup NuGet Credentials - shell: bash - run: | - `vcpkg fetch nuget | tail -n 1` \ - sources add \ - -source "https://nuget.pkg.github.com/BlueQuartzSoftware/index.json" \ - -storepasswordincleartext \ - -name "GitHub" \ - -username "BlueQuartzSoftware" \ - -password "${{secrets.GITHUB_TOKEN}}" - `vcpkg fetch nuget | tail -n 1` \ - setapikey "${{secrets.GITHUB_TOKEN}}" \ - -source "https://nuget.pkg.github.com/BlueQuartzSoftware/index.json" - - name: Install Sphinx - run: | - pip install sphinx myst-parser sphinx-markdown-tables sphinx_rtd_theme numpy - - name: Configure - run: | - cmake --preset ci-windows-${{matrix.toolset}} ${{github.workspace}} -T ${{matrix.toolset}} - - name: Build - run: | - cmake --build --preset ci-windows-${{matrix.toolset}} - - name: Test - run: | - ctest --preset ci-windows-${{matrix.toolset}} diff --git a/CMakePresets.json b/CMakePresets.json index beceb85731..d91617d0c0 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -186,6 +186,92 @@ } } }, + { + "name": "ci-windows-v143-ooc", + "displayName": "ci-windows-v143-ooc", + "description": "Build configuration for Azure Out-of-Core builds", + "generator": "Visual Studio 17 2022", + "inherits": "ci", + "cacheVariables": { + "VCPKG_TARGET_TRIPLET": { + "type": "STRING", + "value": "x64-windows-v143" + }, + "VCPKG_HOST_TRIPLET": { + "type": "STRING", + "value": "x64-windows-v143" + }, + "VCPKG_MANIFEST_FEATURES": { + "type": "STRING", + "value": "python;tests;parallel;itk;ebsd;compressors" + }, + "SIMPLNX_ENABLE_COMPRESSORS": { + "type": "BOOL", + "value": "ON" + }, + "SIMPLNX_EXTRA_PLUGINS": { + "type": "STRING", + "value": "FileStore;SimplnxReview" + }, + "SIMPLNX_FORCE_OUT_OF_CORE_DATA": { + "type": "BOOL", + "value": "ON" + }, + "SIMPLNX_SimplnxReview_SOURCE_DIR": { + "type": "PATH", + "value": "${sourceDir}/SimplnxReview" + }, + "SIMPLNX_FileStore_SOURCE_DIR": { + "type": "PATH", + "value": "${sourceDir}/FileStore" + } + } + }, + { + "name": "ci-linux-x64-ooc", + "displayName": "ci-linux-x64-ooc", + "description": "Build configuration for Azure Out-of-Core builds", + "generator": "Ninja", + "inherits": "ci", + "cacheVariables": { + "VCPKG_TARGET_TRIPLET": { + "type": "STRING", + "value": "x64-linux-dynamic" + }, + "VCPKG_HOST_TRIPLET": { + "type": "STRING", + "value": "x64-linux-dynamic" + }, + "Python3_EXECUTABLE": { + "type": "PATH", + "value": "/opt/local/anaconda3/envs/dream3d/bin/python" + }, + "VCPKG_MANIFEST_FEATURES": { + "type": "STRING", + "value": "python;tests;parallel;itk;ebsd;compressors" + }, + "SIMPLNX_ENABLE_COMPRESSORS": { + "type": "BOOL", + "value": "ON" + }, + "SIMPLNX_EXTRA_PLUGINS": { + "type": "STRING", + "value": "FileStore;SimplnxReview" + }, + "SIMPLNX_FORCE_OUT_OF_CORE_DATA": { + "type": "BOOL", + "value": "ON" + }, + "SIMPLNX_SimplnxReview_SOURCE_DIR": { + "type": "PATH", + "value": "${sourceDir}/SimplnxReview" + }, + "SIMPLNX_FileStore_SOURCE_DIR": { + "type": "PATH", + "value": "${sourceDir}/FileStore" + } + } + }, { "name": "conda", "generator": "Ninja",