Merge remote-tracking branch 'upstream/develop' into stable

* Panasonic DC-GX7MK3 camera support
* SONY ILCE-7C camera support

* upstream/develop:
  Github Actions CI: enable MINGW32 testing
  iPoint2D: `size_t` is not guaranteed to be larger than `int`
  GitHub Actions CI: update to LLVM/Clang 11
  Cr2Decoder::decodeNewFormat(): actually ensure that subsampled image dims are multiples of subsampling factor
  Github Actions CI: Add CodeQL analysis
  Port clang static analysis ci job to github actions/CodeChecker
  Cr2Decoder::decodeNewFormat(): ensure that if subsampling is on, there is subsampling, and vice versa
  AbstractLJpegDecompressor::parseSOF(): don't write mRaw->metadata.subsampling, but check against it
  Cr2Decoder::decodeNewFormat(): store subsampling in RawImage's subsampling field
  rawspeed: SONY ILCE-7C support
  Add DC-GX7MK3 alias & remove spaces
  GHA: try to actually fix rpu-enabled coverage build
  And fix yet another typo in gha yaml
  GH Actions: fix rpu-less build
  Github actions: and don't forget to re-enable linux clang build...
  Github actions: and don't forget to re-enable linux RelWithDebInfo build..
  Fixup github actions workflow yaml
  Port modern gcc/clang/gcc coverage builds to github actions
  README: update for travis-ci org->com migration
  Github Actions CI: retry using better upload name for codecov uploads

.github/workflows/ci.yml

@@ -6,7 +6,185 @@ on:
   pull_request:
     branches: [ develop ]
 
+env:
+  RPUU_DST: ${{ github.workspace }}/../raw-camera-samples/raw.pixls.us-unique
+
 jobs:
+  linux:
+    name: ${{ matrix.os }}.${{ matrix.compiler.compiler }}.${{ matrix.flavor }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ linux ]
+        compiler:
+          - { compiler: GNU,  CC: gcc-10,   CXX: g++-10,     GCOV: gcov-10 }
+          - { compiler: LLVM, CC: clang-11, CXX: clang++-11 }
+        flavor: [ RelWithDebInfo, Coverage, ClangStaticAnalysis, CodeQLAnalysis ]
+        exclude:
+          - os: linux
+            compiler: { compiler: LLVM, CC: clang-11, CXX: clang++-11 }
+            flavor: Coverage
+          - os: linux
+            compiler: { compiler: GNU,  CC: gcc-10,   CXX: g++-10, GCOV: gcov-10 }
+            flavor: ClangStaticAnalysis
+          - os: linux
+            compiler: { compiler: GNU,  CC: gcc-10,   CXX: g++-10, GCOV: gcov-10 }
+            flavor: CodeQLAnalysis
+    container: debian:testing-slim
+    steps:
+      - name: Configure APT
+        env:
+          DEBIAN_FRONTEND: noninteractive
+        run: |
+          # Paper over occasional network flakiness of some mirrors.
+          echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80retry
+          # Do not install recommended packages
+          echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/80recommends
+          # Do not install suggested packages
+          echo 'APT::Install-Suggests "false";' > /etc/apt/apt.conf.d/80suggests
+          # Assume yes
+          echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/80forceyes
+          # Fix broken packages
+          echo 'APT::Get::Fix-Missing "true";' > /etc/apt/apt.conf.d/80fixmissing
+      - name: Install/Update necessary packages
+        run: |
+          rm -rf /var/lib/apt/lists/*;
+          apt-get update;
+          apt-get install clang++-11 \
+                          clang-tidy-11 \
+                          clang-tools-11 \
+                          cmake \
+                          curl \
+                          doxygen \
+                          g++-10 \
+                          gcc \
+                          gcc-10 \
+                          git \
+                          googletest \
+                          graphviz \
+                          libc++-11-dev \
+                          libjpeg-dev \
+                          libomp-11-dev \
+                          libpugixml-dev \
+                          libxml2-utils \
+                          make \
+                          ninja-build \
+                          python3-sphinx \
+                          python3-sphinx-rtd-theme \
+                          rsync \
+                          unzip \
+                          zlib1g-dev;
+          apt-get clean;
+          dpkg-divert --add --rename --divert /usr/bin/ld.original /usr/bin/ld;
+          ln -s /usr/bin/ld.gold /usr/bin/ld
+          ln -s /usr/bin/clang-11 /usr/local/bin/clang
+          ln -s /usr/bin/clang++-11 /usr/local/bin/clang++
+          ln -s /usr/bin/clang-tidy-11 /usr/local/bin/clang-tidy
+      - uses: actions/checkout@v2
+        name: Fetch/Checkout RawSpeed git repo
+        with:
+          path: ''
+      - if: matrix.flavor == 'ClangStaticAnalysis'
+        name: Fetch/Checkout CodeChecker git repo (for static analysis)
+        uses: actions/checkout@v2
+        with:
+          repository: 'Ericsson/codechecker'
+          path: 'codechecker'
+          fetch-depth: '1'
+      - if: matrix.flavor == 'ClangStaticAnalysis'
+        name: Move CodeChecker git repo into right place
+        run: |
+          mv codechecker ..
+      - if: matrix.flavor == 'ClangStaticAnalysis'
+        name: Install CodeChecker (for static analysis)
+        run: |
+          rm -rf /var/lib/apt/lists/*;
+          apt-get update;
+          apt-get install python3-dev \
+                          python3-virtualenv;
+          apt-get clean;
+          cd ../codechecker
+          make venv
+          . $PWD/venv/bin/activate
+          BUILD_LOGGER_64_BIT_ONLY=YES BUILD_UI_DIST=NO make package
+          export PATH="$PWD/build/CodeChecker/bin:$PATH"
+      - if: matrix.flavor == 'CodeQLAnalysis'
+        name: Initialize CodeQL
+        uses: github/codeql-action/init@v1
+        with:
+          languages: cpp
+      - name: Fetch raw.pixls.us masterset digest
+        if: matrix.flavor == 'Coverage'
+        env:
+          RPUU_DST: ${{ env.RPUU_DST }}
+        id: fetch-rpuu-digest
+        run: |
+          mkdir -p ${RPUU_DST}
+          rsync -vvrLcW --preallocate --delete --compress --compress-level=1 --stats --progress rsync://raw.pixls.us/data-unique/filelist.sha1 ${RPUU_DST}/;
+          echo "::set-output name=rpuu-digest-hash::$(sha512sum ${RPUU_DST}/filelist.sha1 | awk '{print $1}')"
+      - name: Cache raw.pixls.us masterset
+        if: matrix.flavor == 'Coverage'
+        uses: actions/cache@v2
+        with:
+          path: ${{ env.RPUU_DST }}
+          key: raw.pixls.us-masterset-${{ steps.fetch-rpuu-digest.outputs.rpuu-digest-hash }}
+      - name: Fetch/update raw.pixls.us masterset
+        if: matrix.flavor == 'Coverage'
+        env:
+          RPUU_DST: ${{ env.RPUU_DST }}
+        run: |
+          rsync -vvrLcW --preallocate --delete --compress --compress-level=1 --stats --progress rsync://raw.pixls.us/data-unique/ ${RPUU_DST}/;
+          cd ${RPUU_DST} && sha1sum -c --strict filelist.sha1;
+      - name: Build And Test
+        env:
+          CC: ${{ matrix.compiler.CC }}
+          CXX: ${{ matrix.compiler.CXX }}
+          GCOV: ${{ matrix.compiler.GCOV }}
+          SRC_DIR: ${{ github.workspace }}
+          BUILD_DIR: ${{ github.workspace }}/../rawspeed-build
+          INSTALL_PREFIX: ${{ github.workspace }}/../rawspeed-install
+          RPUU_DST: ${{ env.RPUU_DST }}
+          ECO: -DALLOW_DOWNLOADING_GOOGLETEST=ON
+          FLAVOR: ${{ matrix.flavor }}
+          TARGET: build
+        run: |
+          if [ ${FLAVOR} = "Coverage" ]; then
+            export ECO="${ECO} -DRAWSPEED_ENABLE_SAMPLE_BASED_TESTING=ON";
+          fi;
+          if [ ${FLAVOR} = "ClangStaticAnalysis" ]; then
+            export ECO="${ECO} -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DUSE_CLANG_TIDY=ON -DCMAKE_C_FLAGS_RELWITHDEBINFO=-UNDEBUG -DCMAKE_CXX_FLAGS_RELWITHDEBINFO=-UNDEBUG";
+          fi;
+          cmake -E make_directory "${BUILD_DIR}";
+          cmake -E make_directory "${INSTALL_PREFIX}";
+          export ECO="${ECO} -DRAWSPEED_REFERENCE_SAMPLE_ARCHIVE=${RPUU_DST}";
+          "${SRC_DIR}/.ci/ci-script.sh";
+      - if: matrix.flavor == 'ClangStaticAnalysis'
+        name: Perform clang static analysis
+        env:
+          SRC_DIR: ${{ github.workspace }}
+          BUILD_DIR: ${{ github.workspace }}/../rawspeed-build
+        run: |
+          . "$PWD/../codechecker/venv/bin/activate"
+          export PATH="$PWD/../codechecker/build/CodeChecker/bin:$PATH"
+          CodeChecker check --logfile "${BUILD_DIR}/compile_commands.json" --analyzers clangsa --jobs "$(nproc --all)" --file "${SRC_DIR}/*" --disable deadcode.DeadStores
+      - if: matrix.flavor == 'CodeQLAnalysis'
+        name: Perform CodeQL analysis
+        uses: github/codeql-action/analyze@v1
+      - if: matrix.flavor == 'Coverage'
+        name: Upload code coverage report
+        env:
+          OS: linux
+          ARCH: x86_64
+          CXX: ${{ matrix.compiler.compiler }}
+          JOB_NAME: ${{ matrix.os }}.${{ matrix.compiler.compiler }}.${{ matrix.flavor }}
+          SRC_DIR: ${{ github.workspace }}
+          BUILD_DIR: ${{ github.workspace }}/../rawspeed-build
+        run: |
+          curl -s https://codecov.io/bash > codecov;
+          chmod +x codecov;
+          ./codecov -e OS,ARCH,CXX -F unittests -n "${JOB_NAME}.Unittests" -R "${SRC_DIR}" -s "${BUILD_DIR}/gcov-reports-unittest" -X gcov -Z;
+          ./codecov -e OS,ARCH,CXX -F integration,rpu_u -n "${JOB_NAME}.Integration" -R "${SRC_DIR}" -s "${BUILD_DIR}/gcov-reports-rsa" -X gcov -Z;
   windows-msys2:
     name: ${{ matrix.os }}.${{ matrix.msys2.msystem }}.${{ matrix.compiler.compiler }}.${{ matrix.flavor }}
     runs-on: ${{ matrix.os }}
@@ -16,7 +194,7 @@ jobs:
         os: [ windows-latest ]
         msys2:
           - { msystem: MINGW64, arch: x86_64 }
-          # - { msystem: MINGW32, arch: i686   } # FIXME
+          - { msystem: MINGW32, arch: i686   }
         compiler:
           - { compiler: GNU,  CC: gcc,   CXX: g++ }
           - { compiler: LLVM, CC: clang, CXX: clang++ }
@@ -30,6 +208,7 @@ jobs:
         shell: msys2 {0}
     steps:
       - uses: actions/checkout@v2
+        name: Fetch/Checkout RawSpeed git repo
       - uses: msys2/setup-msys2@v2
         with:
           msystem: ${{ matrix.msys2.msystem }}
@@ -40,8 +219,8 @@ jobs:
           CC: ${{ matrix.compiler.CC }}
           CXX: ${{ matrix.compiler.CXX }}
           SRC_DIR: ${{ github.workspace }}
-          BUILD_DIR: ${{ github.workspace }}/build
-          INSTALL_PREFIX: ${{ github.workspace }}/install
+          BUILD_DIR: ${{ github.workspace }}/../rawspeed-build
+          INSTALL_PREFIX: ${{ github.workspace }}/../rawspeed-install
           ECO: -DALLOW_DOWNLOADING_GOOGLETEST=ON
           FLAVOR: ${{ matrix.flavor }}
           TARGET: build
@@ -55,9 +234,9 @@ jobs:
           OS: windows
           ARCH: ${{ matrix.msys2.arch }}
           CXX: ${{ matrix.compiler.compiler }}
-          JOB_NAME: ${{ jobs.${{ github.job }}.name }}
+          JOB_NAME: ${{ matrix.os }}.${{ matrix.msys2.msystem }}.${{ matrix.compiler.compiler }}.${{ matrix.flavor }}
           SRC_DIR: ${{ github.workspace }}
-          BUILD_DIR: ${{ github.workspace }}/build
+          BUILD_DIR: ${{ github.workspace }}/../rawspeed-build
         run: |
           curl -s https://codecov.io/bash > codecov;
           chmod +x codecov;
@@ -74,6 +253,7 @@ jobs:
         flavor: [ RelWithDebInfo, Coverage ]
     steps:
       - uses: actions/checkout@v2
+        name: Fetch/Checkout RawSpeed git repo
       - name: Install Base Dependencies
         run: |
           brew update > /dev/null || true
@@ -85,8 +265,8 @@ jobs:
           CC: ${{ matrix.compiler.CC }}
           CXX: ${{ matrix.compiler.CXX }}
           SRC_DIR: ${{ github.workspace }}
-          BUILD_DIR: ${{ github.workspace }}/build
-          INSTALL_PREFIX: ${{ github.workspace }}/install
+          BUILD_DIR: ${{ github.workspace }}/../rawspeed-build
+          INSTALL_PREFIX: ${{ github.workspace }}/../rawspeed-install
           ECO: -DALLOW_DOWNLOADING_GOOGLETEST=ON
           FLAVOR: ${{ matrix.flavor }}
           TARGET: build
@@ -100,9 +280,9 @@ jobs:
           OS: macOS
           ARCH: x86_64
           CXX: AppleXcode
-          JOB_NAME: ${{ jobs.${{ github.job }}.name }}
+          JOB_NAME: ${{ matrix.os }}.${{ matrix.compiler.compiler }}.${{ matrix.flavor }}
           SRC_DIR: ${{ github.workspace }}
-          BUILD_DIR: ${{ github.workspace }}/build
+          BUILD_DIR: ${{ github.workspace }}/../rawspeed-build
         run: |
           curl -s https://codecov.io/bash > codecov;
           chmod +x codecov;

.travis.yml

@@ -36,18 +36,10 @@ install:
       travis_retry sudo apt-get update -q;
       travis_retry sudo apt-get install -y -q -f --fix-missing clang-3.5;
     fi;
-  - if [[ "$TRAVIS_OS_NAME" == "linux" && "$FLAVOR" == "Coverage" ]]; then
-      travis_retry sudo apt-get install -y -q -f --fix-missing rsync;
-    fi;
   - if [[ "$TRAVIS_OS_NAME" == "linux" && "$EXTRA" != "NODOCKER" ]]; then
       travis_retry travis_wait 5 docker pull darktable/rawspeed;
     fi;
 
-before_script:
-  - if [[ "$TRAVIS_OS_NAME" == "linux" && "$FLAVOR" == "Coverage" ]]; then
-      travis_retry rsync -vvrLtW --preallocate --delete --compress --compress-level=1 --stats --progress rsync://raw.pixls.us/data-unique/ $HOME/$RPUU_DST && cd $HOME/$RPUU_DST && sha1sum -c --strict filelist.sha1;
-    fi;
-
 env:
   global:
     # The next declaration is the encrypted COVERITY_SCAN_TOKEN, created
@@ -64,42 +56,13 @@ env:
 jobs:
   fast_finish: true
   include:
-  - stage: test
-    os: linux
-    dist: bionic
-    sudo: required
-    services:
-      - docker
-    env: CC=gcc-10 CXX=g++-10
   - stage: test
     os: linux
     dist: bionic
     sudo: required
     services:
       - docker
     env: CC=gcc-10 CXX=g++-10 ECO="-DWITH_OPENMP=OFF"
-  - os: linux
-    dist: bionic
-    sudo: required
-    services:
-      - docker
-    env: CC=clang-10 CXX=clang++-10
-  - os: linux
-    dist: bionic
-    sudo: required
-    services:
-      - docker
-    env: CC=gcc-10 CXX=g++-10 GCOV=gcov-10 FLAVOR=Coverage ECO="-DRAWSPEED_ENABLE_SAMPLE_BASED_TESTING=ON" EXTRA_OPTS="$DOCKER_RPUU"
-    cache:
-      directories:
-        - '$HOME/$RPUU_DST'
-  - os: linux
-    dist: bionic
-    sudo: required
-    services:
-      - docker
-    # Don't forget to ensure that FindLLVMClangTidy.cmake is also bumped
-    env: CC=clang-10 CXX=clang++-10 TARGET=STATICANALYSIS ECO="-DUSE_CLANG_TIDY=ON"
   - os: linux
     dist: bionic
     sudo: required
@@ -133,8 +96,6 @@ jobs:
       - docker
     env: CC=clang-10 CXX=clang++-10 TARGET=WWW
 
-# linux clang Coverage build OOM's in gcov during final codecov report collection
-
 script:
   - set -e
   - if [[ "$TRAVIS_OS_NAME" == "linux" && "$EXTRA" == "NODOCKER" ]]; then
@@ -157,32 +118,13 @@ script:
 
       export TARGET2="$TARGET";
       export CMD="$SRC_DIR/.ci/ci-script.sh";
-      if [[ "$TARGET" == "STATICANALYSIS" ]]; then
-        export TARGET="build";
-        export CMD="scan-build-10 --use-cc=\"$CC\" --use-c++=\"$CXX\" --force-analyze-debug-code --status-bugs -disable-checker deadcode.DeadStores $CMD";
-      fi;
 
       docker run --read-only --tmpfs /tmp --volume $TRAVIS_BUILD_DIR:$SRC_DIR:ro --volume "$TRAVIS_BUILD_DIR/build":$BUILD_DIR --workdir $BUILD_DIR --tmpfs $INSTALL_PREFIX $EXTRA_TMPFS $EXTRA_OPTS --env CC --env CXX --env GCOV --env CFLAGS --env CXXFLAGS --env SRC_DIR --env BUILD_DIR --env INSTALL_PREFIX --env TARGET --env FLAVOR --env ECO darktable/rawspeed sh -c "$CMD";
 
       export TARGET="$TARGET2";
     fi;
 
 after_success:
-  - if [[ "$FLAVOR" == "Coverage" ]]; then
-      cd "$TRAVIS_BUILD_DIR";
-      curl -s https://codecov.io/bash > codecov;
-      chmod +x codecov;
-      export PLATFORM="$TRAVIS_OS_NAME";
-      if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-        if [[ "$CC" == "cc" ]]; then
-          export CXX=GCC;
-        fi;
-      fi;
-      ./codecov -e PLATFORM,CXX -F unittests -X gcov -s "$TRAVIS_BUILD_DIR/build/gcov-reports-unittest" -Z;
-      if [[ "$TRAVIS_OS_NAME" == "linux" && "$FLAVOR" == "Coverage" ]]; then
-        ./codecov -e PLATFORM,CXX -F integration,rpu_u -X gcov -s "$TRAVIS_BUILD_DIR/build/gcov-reports-rsa" -Z;
-      fi;
-    fi;
   - if [[ "$TRAVIS_OS_NAME" == "linux" && "$TARGET" == "WWW" && "$TRAVIS_PULL_REQUEST" == "false" && "$TRAVIS_SECURE_ENV_VARS" == "true" && "$TRAVIS_BRANCH" == "develop" ]]; then
       "$TRAVIS_BUILD_DIR/.ci/www-finish.sh";
     fi;

README.rst

@@ -3,8 +3,8 @@ rawspeed |github actions| |travis-ci| OBS_ |codecov| |oss-fuzz|
 .. |github actions| image:: https://github.com/darktable-org/rawspeed/workflows/CI/badge.svg
     :target: https://github.com/darktable-org/rawspeed/actions?query=workflow%3ACI
 
-.. |travis-ci| image:: https://travis-ci.org/darktable-org/rawspeed.svg?branch=develop
-    :target: https://travis-ci.org/darktable-org/rawspeed
+.. |travis-ci| image:: https://travis-ci.com/darktable-org/rawspeed.svg?branch=develop
+    :target: https://travis-ci.com/darktable-org/rawspeed
 
 .. _OBS: https://build.opensuse.org/project/monitor/graphics:darktable:master
 

cmake/Modules/FindLLVMClangTidy.cmake

@@ -1,4 +1,4 @@
-find_program(CLANGTIDY_PATH NAMES clang-tidy clang-tidy-10 clang-tidy-9 clang-tidy-8 clang-tidy-7 clang-tidy-6.0 clang-tidy-5.0 clang-tidy-4.0 clang-tidy-3.9)
+find_program(CLANGTIDY_PATH NAMES "$ENV{CLANG_TIDY}" clang-tidy)
 
 include(FindPackageHandleStandardArgs)
 find_package_handle_standard_args(LLVMClangTidy