From d436d825550ccf0d5ec69d068905d0cf035c64c3 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 11 Nov 2024 15:11:13 +0000 Subject: [PATCH 01/11] Add builds for musllinux and aarch64 [cd build] --- .github/workflows/build-python-wheels.yml | 24 +++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-python-wheels.yml b/.github/workflows/build-python-wheels.yml index 705ff65..c5855c4 100644 --- a/.github/workflows/build-python-wheels.yml +++ b/.github/workflows/build-python-wheels.yml @@ -82,18 +82,18 @@ jobs: - os: ubuntu-latest bitness: 32 platform_id: manylinux_i686 - # - os: ubuntu-latest - # bitness: 64 - # platform_id: musllinux_x86_64 - # - os: ubuntu-latest - # bitness: 32 - # platform_id: musllinux_i686 - # - os: ubuntu-latest - # bitness: 64 - # platform_id: manylinux_aarch64 - # - os: ubuntu-latest - # bitness: 64 - # platform_id: musllinux_aarch64 + - os: ubuntu-latest + bitness: 64 + platform_id: musllinux_x86_64 + - os: ubuntu-latest + bitness: 32 + platform_id: musllinux_i686 + - os: ubuntu-latest + bitness: 64 + platform_id: manylinux_aarch64 + - os: ubuntu-latest + bitness: 64 + platform_id: musllinux_aarch64 - os: windows-latest bitness: 64 platform_id: win_amd64 From 77733d719bda0ed731f0a124ca8656a36cb792a9 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 11 Nov 2024 16:25:43 +0000 Subject: [PATCH 02/11] Add Make CIBW_BUILD list steps ([cd build] --- .github/workflows/build-python-wheels.yml | 71 ++++++++++++++--------- 1 file changed, 43 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build-python-wheels.yml b/.github/workflows/build-python-wheels.yml index c5855c4..62b1d0c 100644 --- a/.github/workflows/build-python-wheels.yml +++ b/.github/workflows/build-python-wheels.yml @@ -56,7 +56,7 @@ jobs: # To trigger the build steps, add "[cd build]" to commit message build_wheels: - name: Build ${{ matrix.python-version }}-${{ matrix.platform_id }} + name: Build ${{ matrix.python-version }} on ${{ matrix.os }} ${{ matrix.bitness }}-bit runs-on: ${{ matrix.os }} needs: check_build_trigger if: needs.check_build_trigger.outputs.build @@ -66,8 +66,9 @@ jobs: # For build identier, see: # https://cibuildwheel.pypa.io/en/stable/options/#build-skip matrix: + # See more build combinations at "Make CIBW_BUILD list" steps below os: [macos-latest, ubuntu-latest, windows-latest] - python-version: ["3.13", "3.12", "3.11", "3.10", "3.9", "3.8", "3.7", "pypy3.10", "pypy3.9"] + python-version: ["3.13", "3.12", "3.11", "3.10", "3.9", "3.8", "3.7", "pypy3.10", "pypy3.9", "pypy3.8", "pypy3.7"] bitness: [64, 32] include: - os: macos-latest @@ -76,24 +77,6 @@ jobs: - os: macos-latest bitness: 64 platform_id: macosx_arm64 - - os: ubuntu-latest - bitness: 64 - platform_id: manylinux_x86_64 - - os: ubuntu-latest - bitness: 32 - platform_id: manylinux_i686 - - os: ubuntu-latest - bitness: 64 - platform_id: musllinux_x86_64 - - os: ubuntu-latest - bitness: 32 - platform_id: musllinux_i686 - - os: ubuntu-latest - bitness: 64 - platform_id: manylinux_aarch64 - - os: ubuntu-latest - bitness: 64 - platform_id: musllinux_aarch64 - os: windows-latest bitness: 64 platform_id: win_amd64 @@ -105,15 +88,17 @@ jobs: bitness: 32 - os: macos-latest python-version: "3.7" - - python-version: "pypy3.10" - bitness: 32 - - python-version: "pypy3.9" - bitness: 32 + # - python-version: "pypy3.10" + # bitness: 32 + # - python-version: "pypy3.9" + # bitness: 32 env: CP_VER: "" + CIBW_BUILD: "" + steps: # cibuildwheel needs a specific Python implementation ID - - name: Make Python implementation ID + - name: Make Python implementation ID (non-Windows) id: convert-version-id if: startsWith(matrix.os, 'windows-') == false run: | @@ -139,19 +124,49 @@ jobs: } echo "Python version: $CP_VER" echo "CP_VER=$CP_VER" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + + - name: Make CIBW_BUILD list (non-Linux) + if: startsWith(matrix.os, 'ubuntu-') == false + run: echo "CIBW_BUILD=${{ env.CP_VER }}-${{ matrix.platform_id }}" >> $GITHUB_ENV + + - name: Make CIBW_BUILD list (Linux 64-bit) + if: startsWith(matrix.os, 'ubuntu-') && matrix.bitness == 64 + run: | + CP_VER=${{ env.CP_VER }} + CIBW_BUILD="$CP_VER-manylinux_x86_64 $CP_VER-manylinux_aarch64" + if [[ "$CP_VER" != pp* ]]; then + CIBW_BUILD="$CIBW_BUILD $CP_VER-musllinux_x86_64 $CP_VER-musllinux_aarch64" + fi + echo "CIBW_BUILD: $CIBW_BUILD" + echo "CIBW_BUILD=$CIBW_BUILD" >> $GITHUB_ENV + + - name: Make CIBW_BUILD list (Linux 32-bit) + if: startsWith(matrix.os, 'ubuntu-') && matrix.bitness == 32 + run: | + CP_VER=${{ env.CP_VER }} + CIBW_BUILD="$CP_VER-manylinux_i686" + if [[ "$CP_VER" != pp* ]]; then + CIBW_BUILD="$CIBW_BUILD $CP_VER-musllinux_i686" + fi + echo "CIBW_BUILD: $CIBW_BUILD" + echo "CIBW_BUILD=$CIBW_BUILD" >> $GITHUB_ENV + - name: Checkout source code uses: actions/checkout@v4 + - name: Setup Python uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: "pip" + - name: Install Python dependencies + run: python -m pip install --upgrade pip + - name: Setup Rust toolchain if: startsWith(matrix.os, 'ubuntu-') == false uses: actions-rust-lang/setup-rust-toolchain@v1 # For Linux, Rust will be installed inside a cibuildwheel container later - - name: Install Python dependencies - run: python -m pip install --upgrade pip + - name: Build Python wheels uses: pypa/cibuildwheel@v2.21.3 with: @@ -161,7 +176,6 @@ jobs: CIBW_BUILD_VERBOSITY: 1 # See build selector name at: # https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip - CIBW_BUILD: ${{ env.CP_VER }}-${{ matrix.platform_id }} CIBW_ENVIRONMENT_MACOS: | MACOSX_DEPLOYMENT_TARGET=10.9 PATH="$HOME/.cargo/bin:$PATH" @@ -174,6 +188,7 @@ jobs: # It is needed to install Rust for Linux, # because cibuildwheel on Linux runs inside a container # and the container does not have Rust. + - name: Store artifacts uses: actions/upload-artifact@v3 with: From 1f44a618f9fafb936db99ee8060d458c0f58d2c7 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 11 Nov 2024 16:38:05 +0000 Subject: [PATCH 03/11] Exclude pypy3.7 on macOS and add CIBW_ARCHS --- .github/workflows/build-python-wheels.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-python-wheels.yml b/.github/workflows/build-python-wheels.yml index 62b1d0c..e059562 100644 --- a/.github/workflows/build-python-wheels.yml +++ b/.github/workflows/build-python-wheels.yml @@ -88,6 +88,8 @@ jobs: bitness: 32 - os: macos-latest python-version: "3.7" + - os: macos-latest + python-version: "pypy3.7" # - python-version: "pypy3.10" # bitness: 32 # - python-version: "pypy3.9" @@ -145,9 +147,6 @@ jobs: run: | CP_VER=${{ env.CP_VER }} CIBW_BUILD="$CP_VER-manylinux_i686" - if [[ "$CP_VER" != pp* ]]; then - CIBW_BUILD="$CIBW_BUILD $CP_VER-musllinux_i686" - fi echo "CIBW_BUILD: $CIBW_BUILD" echo "CIBW_BUILD=$CIBW_BUILD" >> $GITHUB_ENV @@ -176,11 +175,13 @@ jobs: CIBW_BUILD_VERBOSITY: 1 # See build selector name at: # https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip + CIBW_ARCHS_MACOS: "x86_64 universal2 arm64" CIBW_ENVIRONMENT_MACOS: | MACOSX_DEPLOYMENT_TARGET=10.9 PATH="$HOME/.cargo/bin:$PATH" CC=/usr/bin/clang CXX=/usr/bin/clang++ + CIBW_ARCHS_LINUX: "auto aarch64" CIBW_ENVIRONMENT_LINUX: PATH="$HOME/.cargo/bin:$PATH" CIBW_BEFORE_BUILD_LINUX: | pip install --upgrade setuptools-rust From d4010997b48dfcba4c552f3429b016bb51233f5e Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 11 Nov 2024 16:39:11 +0000 Subject: [PATCH 04/11] [cd build] --- .github/workflows/build-python-wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-python-wheels.yml b/.github/workflows/build-python-wheels.yml index e059562..d8a0ec2 100644 --- a/.github/workflows/build-python-wheels.yml +++ b/.github/workflows/build-python-wheels.yml @@ -173,7 +173,7 @@ jobs: output-dir: wheelhouse env: CIBW_BUILD_VERBOSITY: 1 - # See build selector name at: + # See CIBW_ARCHS and other build selectors at: # https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip CIBW_ARCHS_MACOS: "x86_64 universal2 arm64" CIBW_ENVIRONMENT_MACOS: | From 0f6129c4733550971554cf5802d426498d2ddef1 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 11 Nov 2024 16:56:55 +0000 Subject: [PATCH 05/11] Use CIBW_BUILD wildcard for macosx builds [cd build] --- .github/workflows/build-python-wheels.yml | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-python-wheels.yml b/.github/workflows/build-python-wheels.yml index d8a0ec2..8207393 100644 --- a/.github/workflows/build-python-wheels.yml +++ b/.github/workflows/build-python-wheels.yml @@ -71,12 +71,6 @@ jobs: python-version: ["3.13", "3.12", "3.11", "3.10", "3.9", "3.8", "3.7", "pypy3.10", "pypy3.9", "pypy3.8", "pypy3.7"] bitness: [64, 32] include: - - os: macos-latest - bitness: 64 - platform_id: macosx_x86_64 - - os: macos-latest - bitness: 64 - platform_id: macosx_arm64 - os: windows-latest bitness: 64 platform_id: win_amd64 @@ -90,10 +84,6 @@ jobs: python-version: "3.7" - os: macos-latest python-version: "pypy3.7" - # - python-version: "pypy3.10" - # bitness: 32 - # - python-version: "pypy3.9" - # bitness: 32 env: CP_VER: "" CIBW_BUILD: "" @@ -113,6 +103,7 @@ jobs: fi echo "Python version: $CP_VER" echo "CP_VER=$CP_VER" >> $GITHUB_ENV + - name: Make Python implementation ID (Windows) id: convert-version-id-win if: startsWith(matrix.os, 'windows-') @@ -127,9 +118,9 @@ jobs: echo "Python version: $CP_VER" echo "CP_VER=$CP_VER" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - - name: Make CIBW_BUILD list (non-Linux) - if: startsWith(matrix.os, 'ubuntu-') == false - run: echo "CIBW_BUILD=${{ env.CP_VER }}-${{ matrix.platform_id }}" >> $GITHUB_ENV + - name: Make CIBW_BUILD list (macOS) + if: startsWith(matrix.os, 'macos-') + run: echo "CIBW_BUILD=${{ env.CP_VER }}-macosx*" >> $GITHUB_ENV - name: Make CIBW_BUILD list (Linux 64-bit) if: startsWith(matrix.os, 'ubuntu-') && matrix.bitness == 64 @@ -150,6 +141,10 @@ jobs: echo "CIBW_BUILD: $CIBW_BUILD" echo "CIBW_BUILD=$CIBW_BUILD" >> $GITHUB_ENV + - name: Make CIBW_BUILD list (Windows) + if: startsWith(matrix.os, 'windows-') + run: echo "CIBW_BUILD=${{ env.CP_VER }}-${{ matrix.platform_id }}" >> $GITHUB_ENV + - name: Checkout source code uses: actions/checkout@v4 From edf9fbf461b7a1507e009ce7c3263c32eb96fab2 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 11 Nov 2024 17:02:35 +0000 Subject: [PATCH 06/11] Add rustup target add x86_64-apple-darwin [cd build] --- .github/workflows/build-python-wheels.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-python-wheels.yml b/.github/workflows/build-python-wheels.yml index 8207393..455b4ad 100644 --- a/.github/workflows/build-python-wheels.yml +++ b/.github/workflows/build-python-wheels.yml @@ -161,6 +161,11 @@ jobs: uses: actions-rust-lang/setup-rust-toolchain@v1 # For Linux, Rust will be installed inside a cibuildwheel container later + - name: Setup rustup target + if: startsWith(matrix.os, 'macos-') + run: rustup target add x86_64-apple-darwin + # For cross-compile x86 on GitHub arm64 runner + - name: Build Python wheels uses: pypa/cibuildwheel@v2.21.3 with: From 0f1e6e97d7a948c5c21783378aa7017066148406 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 11 Nov 2024 18:26:45 +0000 Subject: [PATCH 07/11] Let cibuildwheel decide the platforms to build [cd build] --- .github/workflows/build-python-wheels.yml | 92 ++++++++++++----------- 1 file changed, 47 insertions(+), 45 deletions(-) diff --git a/.github/workflows/build-python-wheels.yml b/.github/workflows/build-python-wheels.yml index 455b4ad..7a96218 100644 --- a/.github/workflows/build-python-wheels.yml +++ b/.github/workflows/build-python-wheels.yml @@ -56,7 +56,7 @@ jobs: # To trigger the build steps, add "[cd build]" to commit message build_wheels: - name: Build ${{ matrix.python-version }} on ${{ matrix.os }} ${{ matrix.bitness }}-bit + name: Build ${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} needs: check_build_trigger if: needs.check_build_trigger.outputs.build @@ -68,25 +68,26 @@ jobs: matrix: # See more build combinations at "Make CIBW_BUILD list" steps below os: [macos-latest, ubuntu-latest, windows-latest] - python-version: ["3.13", "3.12", "3.11", "3.10", "3.9", "3.8", "3.7", "pypy3.10", "pypy3.9", "pypy3.8", "pypy3.7"] - bitness: [64, 32] - include: - - os: windows-latest - bitness: 64 - platform_id: win_amd64 - - os: windows-latest - bitness: 32 - platform_id: win32 - exclude: - - os: macos-latest - bitness: 32 - - os: macos-latest - python-version: "3.7" - - os: macos-latest - python-version: "pypy3.7" + python-version: ["3.13"] + # python-version: ["3.13", "3.12", "3.11", "3.10", "3.9", "3.8", "3.7", "pypy3.10", "pypy3.9", "pypy3.8", "pypy3.7"] + # bitness: [64, 32] + # include: + # - os: windows-latest + # bitness: 64 + # platform_id: win_amd64 + # - os: windows-latest + # bitness: 32 + # platform_id: win32 + # exclude: + # - os: macos-latest + # bitness: 32 + # - os: macos-latest + # python-version: "3.7" + # - os: macos-latest + # python-version: "pypy3.7" env: CP_VER: "" - CIBW_BUILD: "" + CIBW_BUILD: "" # let cibuildwheel cover the supported platforms steps: # cibuildwheel needs a specific Python implementation ID @@ -118,32 +119,32 @@ jobs: echo "Python version: $CP_VER" echo "CP_VER=$CP_VER" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - - name: Make CIBW_BUILD list (macOS) - if: startsWith(matrix.os, 'macos-') - run: echo "CIBW_BUILD=${{ env.CP_VER }}-macosx*" >> $GITHUB_ENV - - - name: Make CIBW_BUILD list (Linux 64-bit) - if: startsWith(matrix.os, 'ubuntu-') && matrix.bitness == 64 - run: | - CP_VER=${{ env.CP_VER }} - CIBW_BUILD="$CP_VER-manylinux_x86_64 $CP_VER-manylinux_aarch64" - if [[ "$CP_VER" != pp* ]]; then - CIBW_BUILD="$CIBW_BUILD $CP_VER-musllinux_x86_64 $CP_VER-musllinux_aarch64" - fi - echo "CIBW_BUILD: $CIBW_BUILD" - echo "CIBW_BUILD=$CIBW_BUILD" >> $GITHUB_ENV + # - name: Make CIBW_BUILD list (macOS) + # if: startsWith(matrix.os, 'macos-') + # run: echo "CIBW_BUILD=${{ env.CP_VER }}-macosx*" >> $GITHUB_ENV + + # - name: Make CIBW_BUILD list (Linux 64-bit) + # if: startsWith(matrix.os, 'ubuntu-') && matrix.bitness == 64 + # run: | + # CP_VER=${{ env.CP_VER }} + # CIBW_BUILD="$CP_VER-manylinux_x86_64" + # if [[ "$CP_VER" != pp* ]]; then + # CIBW_BUILD="$CIBW_BUILD $CP_VER-musllinux_x86_64" + # fi + # echo "CIBW_BUILD: $CIBW_BUILD" + # echo "CIBW_BUILD=$CIBW_BUILD" >> $GITHUB_ENV - - name: Make CIBW_BUILD list (Linux 32-bit) - if: startsWith(matrix.os, 'ubuntu-') && matrix.bitness == 32 - run: | - CP_VER=${{ env.CP_VER }} - CIBW_BUILD="$CP_VER-manylinux_i686" - echo "CIBW_BUILD: $CIBW_BUILD" - echo "CIBW_BUILD=$CIBW_BUILD" >> $GITHUB_ENV - - - name: Make CIBW_BUILD list (Windows) - if: startsWith(matrix.os, 'windows-') - run: echo "CIBW_BUILD=${{ env.CP_VER }}-${{ matrix.platform_id }}" >> $GITHUB_ENV + # - name: Make CIBW_BUILD list (Linux 32-bit) + # if: startsWith(matrix.os, 'ubuntu-') && matrix.bitness == 32 + # run: | + # CP_VER=${{ env.CP_VER }} + # CIBW_BUILD="$CP_VER-manylinux_i686" + # echo "CIBW_BUILD: $CIBW_BUILD" + # echo "CIBW_BUILD=$CIBW_BUILD" >> $GITHUB_ENV + + # - name: Make CIBW_BUILD list (Windows) + # if: startsWith(matrix.os, 'windows-') + # run: CIBW_BUILD=${{ env.CP_VER }}-${{ matrix.platform_id }}" >> $GITHUB_ENV - name: Checkout source code uses: actions/checkout@v4 @@ -175,13 +176,13 @@ jobs: CIBW_BUILD_VERBOSITY: 1 # See CIBW_ARCHS and other build selectors at: # https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip - CIBW_ARCHS_MACOS: "x86_64 universal2 arm64" + CIBW_ARCHS_MACOS: "x86_64 arm64" CIBW_ENVIRONMENT_MACOS: | MACOSX_DEPLOYMENT_TARGET=10.9 PATH="$HOME/.cargo/bin:$PATH" CC=/usr/bin/clang CXX=/usr/bin/clang++ - CIBW_ARCHS_LINUX: "auto aarch64" + CIBW_ARCHS_LINUX: "auto" CIBW_ENVIRONMENT_LINUX: PATH="$HOME/.cargo/bin:$PATH" CIBW_BEFORE_BUILD_LINUX: | pip install --upgrade setuptools-rust @@ -189,6 +190,7 @@ jobs: # It is needed to install Rust for Linux, # because cibuildwheel on Linux runs inside a container # and the container does not have Rust. + CIBW_ARCHS_WINDOWS: "AMD64 x86" - name: Store artifacts uses: actions/upload-artifact@v3 From 6de8dd80335fa379c68875551676bce8199ab46b Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 11 Nov 2024 19:04:50 +0000 Subject: [PATCH 08/11] CIBW_SKIP: "cp37-musllinux_i686" [cd build] --- .github/workflows/build-python-wheels.yml | 81 +---------------------- 1 file changed, 3 insertions(+), 78 deletions(-) diff --git a/.github/workflows/build-python-wheels.yml b/.github/workflows/build-python-wheels.yml index 7a96218..66f9a1a 100644 --- a/.github/workflows/build-python-wheels.yml +++ b/.github/workflows/build-python-wheels.yml @@ -63,89 +63,13 @@ jobs: strategy: # Ensure that a wheel builder finishes even if another fails fail-fast: false - # For build identier, see: - # https://cibuildwheel.pypa.io/en/stable/options/#build-skip matrix: - # See more build combinations at "Make CIBW_BUILD list" steps below os: [macos-latest, ubuntu-latest, windows-latest] python-version: ["3.13"] - # python-version: ["3.13", "3.12", "3.11", "3.10", "3.9", "3.8", "3.7", "pypy3.10", "pypy3.9", "pypy3.8", "pypy3.7"] - # bitness: [64, 32] - # include: - # - os: windows-latest - # bitness: 64 - # platform_id: win_amd64 - # - os: windows-latest - # bitness: 32 - # platform_id: win32 - # exclude: - # - os: macos-latest - # bitness: 32 - # - os: macos-latest - # python-version: "3.7" - # - os: macos-latest - # python-version: "pypy3.7" env: - CP_VER: "" - CIBW_BUILD: "" # let cibuildwheel cover the supported platforms + CIBW_BUILD: "" # blank, let cibuildwheel build all supported platforms steps: - # cibuildwheel needs a specific Python implementation ID - - name: Make Python implementation ID (non-Windows) - id: convert-version-id - if: startsWith(matrix.os, 'windows-') == false - run: | - PYTHON_VERSION=${{ matrix.python-version }} - if [[ "$PYTHON_VERSION" == pypy* ]]; then - CP_VER="pp${PYTHON_VERSION:4}" - CP_VER="${CP_VER//./}" - else - CP_VER="cp${PYTHON_VERSION//./}" - fi - echo "Python version: $CP_VER" - echo "CP_VER=$CP_VER" >> $GITHUB_ENV - - - name: Make Python implementation ID (Windows) - id: convert-version-id-win - if: startsWith(matrix.os, 'windows-') - shell: powershell - run: | - $PYTHON_VERSION = "${{ matrix.python-version }}" - if ($PYTHON_VERSION -like "pypy*") { - $CP_VER = "pp" + $PYTHON_VERSION.Substring(4).Replace(".", "") - } else { - $CP_VER = "cp" + $PYTHON_VERSION.Replace(".", "") - } - echo "Python version: $CP_VER" - echo "CP_VER=$CP_VER" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - - # - name: Make CIBW_BUILD list (macOS) - # if: startsWith(matrix.os, 'macos-') - # run: echo "CIBW_BUILD=${{ env.CP_VER }}-macosx*" >> $GITHUB_ENV - - # - name: Make CIBW_BUILD list (Linux 64-bit) - # if: startsWith(matrix.os, 'ubuntu-') && matrix.bitness == 64 - # run: | - # CP_VER=${{ env.CP_VER }} - # CIBW_BUILD="$CP_VER-manylinux_x86_64" - # if [[ "$CP_VER" != pp* ]]; then - # CIBW_BUILD="$CIBW_BUILD $CP_VER-musllinux_x86_64" - # fi - # echo "CIBW_BUILD: $CIBW_BUILD" - # echo "CIBW_BUILD=$CIBW_BUILD" >> $GITHUB_ENV - - # - name: Make CIBW_BUILD list (Linux 32-bit) - # if: startsWith(matrix.os, 'ubuntu-') && matrix.bitness == 32 - # run: | - # CP_VER=${{ env.CP_VER }} - # CIBW_BUILD="$CP_VER-manylinux_i686" - # echo "CIBW_BUILD: $CIBW_BUILD" - # echo "CIBW_BUILD=$CIBW_BUILD" >> $GITHUB_ENV - - # - name: Make CIBW_BUILD list (Windows) - # if: startsWith(matrix.os, 'windows-') - # run: CIBW_BUILD=${{ env.CP_VER }}-${{ matrix.platform_id }}" >> $GITHUB_ENV - - name: Checkout source code uses: actions/checkout@v4 @@ -174,8 +98,9 @@ jobs: output-dir: wheelhouse env: CIBW_BUILD_VERBOSITY: 1 - # See CIBW_ARCHS and other build selectors at: + # See CIBW_BUILD, CIBW_SKIP, CIBW_ARCHS and other build selectors at: # https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip + CIBW_SKIP: "cp37-musllinux_i686" CIBW_ARCHS_MACOS: "x86_64 arm64" CIBW_ENVIRONMENT_MACOS: | MACOSX_DEPLOYMENT_TARGET=10.9 From 3cccf010863a67b4701ca3d77aee1762db6f7b69 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 11 Nov 2024 19:24:20 +0000 Subject: [PATCH 09/11] Skip *-musllinux_i686 [cd build] --- .github/workflows/build-python-wheels.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-python-wheels.yml b/.github/workflows/build-python-wheels.yml index 66f9a1a..11df057 100644 --- a/.github/workflows/build-python-wheels.yml +++ b/.github/workflows/build-python-wheels.yml @@ -1,7 +1,7 @@ -# Build wheels for many platforms +# Build wheels for many platforms, use cibuildwheel # see: https://github.com/pypa/cibuildwheel -name: Build Python wheels +name: Build wheels on: push: @@ -27,7 +27,7 @@ on: jobs: echo_github_env: - name: Echo GitHub environment variables + name: Echo env variables runs-on: ubuntu-latest steps: - run: | @@ -56,7 +56,7 @@ jobs: # To trigger the build steps, add "[cd build]" to commit message build_wheels: - name: Build ${{ matrix.python-version }} on ${{ matrix.os }} + name: Build ${{ matrix.os }} runs-on: ${{ matrix.os }} needs: check_build_trigger if: needs.check_build_trigger.outputs.build @@ -100,7 +100,7 @@ jobs: CIBW_BUILD_VERBOSITY: 1 # See CIBW_BUILD, CIBW_SKIP, CIBW_ARCHS and other build selectors at: # https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip - CIBW_SKIP: "cp37-musllinux_i686" + CIBW_SKIP: "*-musllinux_i686" CIBW_ARCHS_MACOS: "x86_64 arm64" CIBW_ENVIRONMENT_MACOS: | MACOSX_DEPLOYMENT_TARGET=10.9 @@ -115,7 +115,7 @@ jobs: # It is needed to install Rust for Linux, # because cibuildwheel on Linux runs inside a container # and the container does not have Rust. - CIBW_ARCHS_WINDOWS: "AMD64 x86" + CIBW_ARCHS_WINDOWS: "AMD64 x86 ARM64" - name: Store artifacts uses: actions/upload-artifact@v3 @@ -123,7 +123,7 @@ jobs: path: ./wheelhouse/*.whl build_sdist: - name: Build Python source distribution + name: Build source distribution runs-on: ubuntu-latest needs: check_build_trigger if: needs.check_build_trigger.outputs.build @@ -145,13 +145,13 @@ jobs: path: nlpo3-python/dist/*.tar.gz publish_pypi: - name: Publish Python package to PyPI + name: Publish to PyPI runs-on: ubuntu-latest needs: [build_wheels, build_sdist] - # Upload to PyPI on every tag starting with 'v' - #if: github.event_name == 'push' && startsWith(github.event.ref, 'v') - # Alternatively, to publish when a GitHub Release is created, use the following rule: + # Publish when a GitHub Release is created: if: github.event_name == 'release' && github.event.action == 'published' + # Alternatively, upload to PyPI on every tag starting with 'v': + #if: github.event_name == 'push' && startsWith(github.event.ref, 'v') steps: - name: Retrieve artifacts uses: actions/download-artifact@v3 From 8f10a60ecf6ae0fdf828d9cccff226d79a0464a0 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 11 Nov 2024 19:45:46 +0000 Subject: [PATCH 10/11] Remove Windows ARM64 [cd build] --- .github/workflows/build-python-wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-python-wheels.yml b/.github/workflows/build-python-wheels.yml index 11df057..5ad4449 100644 --- a/.github/workflows/build-python-wheels.yml +++ b/.github/workflows/build-python-wheels.yml @@ -115,7 +115,7 @@ jobs: # It is needed to install Rust for Linux, # because cibuildwheel on Linux runs inside a container # and the container does not have Rust. - CIBW_ARCHS_WINDOWS: "AMD64 x86 ARM64" + CIBW_ARCHS_WINDOWS: "AMD64 x86" - name: Store artifacts uses: actions/upload-artifact@v3 From 7f82ba6799ccbca057c6cefa67eae9877013db91 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 11 Nov 2024 21:23:41 +0000 Subject: [PATCH 11/11] Add binary wheels table --- README.md | 50 ++++++------ nlpo3-python/README.md | 146 +++++++++++++++++++++++++++++------- nlpo3-python/pyproject.toml | 2 + 3 files changed, 146 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index bae6f95..aa45578 100644 --- a/README.md +++ b/README.md @@ -25,14 +25,14 @@ pip install nlpo3 ## Table of contents - [Features](#features) -- [Dictionary file](#dictionary-file) -- [Usage](#usage) +- [Use](#use) - [Node.js binding](#nodejs-binding) - [Python binding](#python-binding) - [Rust library](#rust-library) - [Command-line interface](#command-line-interface) + - [Dictionary](#dictionary) - [Build](#build) -- [Development](#development) +- [Develop](#develop) - [License](#license) ## Features @@ -48,25 +48,7 @@ pip install nlpo3 [tcc]: https://dl.acm.org/doi/10.1145/355214.355225 [benchmark]: ./nlpo3-python/notebooks/nlpo3_segment_benchmarks.ipynb -## Dictionary file - -- For the interest of library size, nlpO3 does not assume what dictionary the - user would like to use, and it does not come with a dictionary. -- A dictionary is needed for the dictionary-based word tokenizer. -- For tokenization dictionary, try - - [words_th.tx][dict-pythainlp] from [PyThaiNLP][pythainlp] - - ~62,000 words - - CC0-1.0 - - [word break dictionary][dict-libthai] from [libthai][libthai] - - consists of dictionaries in different categories, with a make script - - LGPL-2.1 - -[pythainlp]: https://github.com/PyThaiNLP/pythainlp -[libthai]: https://github.com/tlwg/libthai/ -[dict-pythainlp]: https://github.com/PyThaiNLP/pythainlp/blob/dev/pythainlp/corpus/words_th.txt -[dict-libthai]: https://github.com/tlwg/libthai/tree/master/data - -## Usage +## Use ### Node.js binding @@ -151,6 +133,24 @@ echo "ฉันกินข้าว" | nlpo3 segment See more at [nlpo3-cli](./nlpo3-cli/). +### Dictionary + +- For the interest of library size, nlpO3 does not assume what dictionary the + user would like to use, and it does not come with a dictionary. +- A dictionary is needed for the dictionary-based word tokenizer. +- For tokenization dictionary, try + - [words_th.tx][dict-pythainlp] from [PyThaiNLP][pythainlp] + - ~62,000 words + - CC0-1.0 + - [word break dictionary][dict-libthai] from [libthai][libthai] + - consists of dictionaries in different categories, with a make script + - LGPL-2.1 + +[pythainlp]: https://github.com/PyThaiNLP/pythainlp +[libthai]: https://github.com/tlwg/libthai/ +[dict-pythainlp]: https://github.com/PyThaiNLP/pythainlp/blob/dev/pythainlp/corpus/words_th.txt +[dict-libthai]: https://github.com/tlwg/libthai/tree/master/data + ## Build ### Requirements @@ -179,13 +179,13 @@ cargo build --release Check `target/` for build artifacts. -## Development +## Develop -Development document: +### Development document - [Notes on custom string](src/NOTE_ON_STRING.md) -Issues: +### Issues - Please report issues at diff --git a/nlpo3-python/README.md b/nlpo3-python/README.md index 53f8032..00b0d98 100644 --- a/nlpo3-python/README.md +++ b/nlpo3-python/README.md @@ -11,6 +11,22 @@ SPDX-License-Identifier: Apache-2.0 Python binding for nlpO3, a Thai natural language processing library in Rust. +To install: + +```bash +pip install nlpo3 +``` + +## Table of Contents + +- [Features](#features) +- [Use](#use) + - [Dictionary](#dictionary) +- [Build](#build) +- [Issues](#issues) +- [License](#license) +- [Binary wheels](#binary-wheels) + ## Features - Thai word tokenizer @@ -24,31 +40,7 @@ Python binding for nlpO3, a Thai natural language processing library in Rust. [tcc]: https://dl.acm.org/doi/10.1145/355214.355225 [benchmark]: ./notebooks/nlpo3_segment_benchmarks.ipynb -## Dictionary file - -- For the interest of library size, nlpO3 does not assume what dictionary the - user would like to use, and it does not come with a dictionary. -- A dictionary is needed for the dictionary-based word tokenizer. -- For tokenization dictionary, try - - [words_th.txt][dict-pythainlp] from [PyThaiNLP][pythainlp] - - ~62,000 words - - CC0-1.0 - - [word break dictionary][dict-libthai] from [libthai][libthai] - - consists of dictionaries in different categories, with a make script - - LGPL-2.1 - -[pythainlp]: https://github.com/PyThaiNLP/pythainlp -[libthai]: https://github.com/tlwg/libthai/ -[dict-pythainlp]: https://github.com/PyThaiNLP/pythainlp/blob/dev/pythainlp/corpus/words_th.txt -[dict-libthai]: https://github.com/tlwg/libthai/tree/master/data - -## Install - -```bash -pip install nlpo3 -``` - -## Usage +## Use Load file `path/to/dict.file` to memory and assign a name `dict_name` to it. @@ -83,6 +75,24 @@ for text with lots of ambiguous word boundaries: segment("สวัสดีครับ", dict_name="dict_name", safe=True) ``` +### Dictionary + +- For the interest of library size, nlpO3 does not assume what dictionary the + user would like to use, and it does not come with a dictionary. +- A dictionary is needed for the dictionary-based word tokenizer. +- For tokenization dictionary, try + - [words_th.txt][dict-pythainlp] from [PyThaiNLP][pythainlp] + - ~62,000 words + - CC0-1.0 + - [word break dictionary][dict-libthai] from [libthai][libthai] + - consists of dictionaries in different categories, with a make script + - LGPL-2.1 + +[pythainlp]: https://github.com/PyThaiNLP/pythainlp +[libthai]: https://github.com/tlwg/libthai/ +[dict-pythainlp]: https://github.com/PyThaiNLP/pythainlp/blob/dev/pythainlp/corpus/words_th.txt +[dict-libthai]: https://github.com/tlwg/libthai/tree/master/data + ## Build ### Requirements @@ -111,9 +121,9 @@ To install a wheel from a local directory: pip install dist/nlpo3-1.3.1-cp311-cp311-macosx_12_0_x86_64.whl ``` -## Test +### Test -To run the Python unit test: +To run a Python unit test: ```bash cd tests @@ -129,3 +139,85 @@ Please report issues at nlpO3 Python binding is copyrighted by its authors and licensed under terms of the Apache Software License 2.0 (Apache-2.0). See file [LICENSE](./LICENSE) for details. + +## Binary wheels + +A pre-built binary package is available from [PyPI][pypi] for these platforms: + +[pypi]: https://pypi.org/project/nlpo3/ + +|Python|OS|Architecture|Has binary wheel?| +|-|-|-|-| +|3.13|Windows|x86|✅| +||Windows|AMD64|✅| +||macOS|x86_64|✅| +||macOS|arm64|✅| +||manylinux|x86_64|✅| +||manylinux|i686|✅| +||musllinux|x86_64|✅| +|3.12|Windows|x86|✅| +||Windows|AMD64|✅| +||macOS|x86_64|✅| +||macOS|arm64|✅| +||manylinux|x86_64|✅| +||manylinux|i686|✅| +||musllinux|x86_64|✅| +|3.11|Windows|x86|✅| +||Windows|AMD64|✅| +||macOS|x86_64|✅| +||macOS|arm64|✅| +||manylinux|x86_64|✅| +||manylinux|i686|✅| +||musllinux|x86_64|✅| +|3.10|Windows|x86|✅| +||Windows|AMD64|✅| +||macOS|x86_64|✅| +||macOS|arm64|✅| +||manylinux|x86_64|✅| +||manylinux|i686|✅| +||musllinux|x86_64|✅| +|3.9|Windows|x86|✅| +||Windows|AMD64|✅| +||macOS|x86_64|✅| +||macOS|arm64|✅| +||manylinux|x86_64|✅| +||manylinux|i686|✅| +||musllinux|x86_64|✅| +|3.8|Windows|x86|✅| +||Windows|AMD64|✅| +||macOS|x86_64|✅| +||macOS|arm64|✅| +||manylinux|x86_64|✅| +||manylinux|i686|✅| +||musllinux|x86_64|✅| +|3.7|Windows|x86|✅| +||Windows|AMD64|✅| +||macOS|x86_64|✅| +||macOS|arm64|❌| +||manylinux|x86_64|✅| +||manylinux|i686|✅| +||musllinux|x86_64|✅| +|PyPy 3.10|Windows|x86|❌| +||Windows|AMD64|✅| +||macOS|x86_64|✅| +||macOS|arm64|✅| +||manylinux|x86_64|✅| +||manylinux|i686|✅| +|PyPy 3.9|Windows|x86|❌| +||Windows|AMD64|✅| +||macOS|x86_64|✅| +||macOS|arm64|✅| +||manylinux|x86_64|✅| +||manylinux|i686|✅| +|PyPy 3.8|Windows|x86|❌| +||Windows|AMD64|✅| +||macOS|x86_64|✅| +||macOS|arm64|✅| +||manylinux|x86_64|✅| +||manylinux|i686|✅| +|PyPy 3.7|Windows|x86|❌| +||Windows|AMD64|✅| +||macOS|x86_64|✅| +||macOS|arm64|❌| +||manylinux|x86_64|✅| +||manylinux|i686|✅| diff --git a/nlpo3-python/pyproject.toml b/nlpo3-python/pyproject.toml index edb6711..e31f84b 100644 --- a/nlpo3-python/pyproject.toml +++ b/nlpo3-python/pyproject.toml @@ -25,6 +25,8 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Natural Language :: Thai",