Use pyvips for image resizing (#185)

* perf: replace PIL with pyvips for 3.3x faster image processing

- Replace PIL with pyvips in image_crops.py for faster image resizing and cropping
- Add pyvips dependency to requirements.txt
- Benchmark shows 3.3x speedup with identical output quality
- Note: requires libvips system package

* Use pyvips-binary instead of system libvips

* Add tests for image_crops and GitHub workflow

* Apply black formatting

* Use improved numpy integration from pyvips 2.2

---------

Co-authored-by: openhands <openhands@all-hands.dev>

.github/workflows/test.yml

@@ -0,0 +1,29 @@
+name: Tests
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pytest
+        pip install -r requirements.txt
+    - name: Run tests
+      run: |
+        python -m pytest tests/test_image_crops.py -v

moondream/torch/image_crops.py

@@ -1,9 +1,9 @@
 import math
 import numpy as np
 import torch
+import pyvips
 
 from typing import TypedDict
-from PIL import Image
 
 
 def select_tiling(
@@ -108,18 +108,20 @@ def overlap_crop_image(
         tiling[1] * crop_window_size + total_margin_pixels,
     )
 
-    # Convert to PIL for resizing
-    pil_image = Image.fromarray(image)
-    resized = pil_image.resize(
-        (target_size[1], target_size[0]), Image.Resampling.LANCZOS
-    )
-    image = np.array(resized)
+    # Convert to vips for resizing
+    vips_image = pyvips.Image.new_from_array(image)
+
+    # Resize using vips
+    scale_x = target_size[1] / image.shape[1]
+    scale_y = target_size[0] / image.shape[0]
+    resized = vips_image.resize(scale_x, vscale=scale_y)
+    image = resized.numpy()
 
     # Create global crop
-    global_pil = pil_image.resize(
-        (base_size[1], base_size[0]), Image.Resampling.LANCZOS
-    )  # PIL uses (width, height)
-    global_crop = np.array(global_pil)
+    scale_x = base_size[1] / vips_image.width
+    scale_y = base_size[0] / vips_image.height
+    global_vips = vips_image.resize(scale_x, vscale=scale_y)
+    global_crop = global_vips.numpy()
 
     # Extract crops with overlap
     crops = []

requirements.txt

@@ -1,6 +1,8 @@
 accelerate==0.32.1
 huggingface-hub==0.24.0
 Pillow==10.4.0
+pyvips-binary==8.16.0
+pyvips==2.2.3
 torch==2.3.1
 torchvision==0.18.1
 transformers==4.44.0

tests/test_image_crops.py

@@ -0,0 +1,58 @@
+import numpy as np
+import torch
+from moondream.torch.image_crops import overlap_crop_image, reconstruct_from_crops
+
+
+def test_overlap_crop_basic():
+    # Create a test image
+    test_image = np.zeros((800, 600, 3), dtype=np.uint8)
+    # Add a recognizable pattern - white rectangle in the middle
+    test_image[300:500, 200:400] = 255
+
+    result = overlap_crop_image(test_image, overlap_margin=4, max_crops=12)
+
+    # Check basic properties
+    assert result["global_crop"].shape == (378, 378, 3)
+    assert len(result["local_crops"]) > 0
+    assert all(crop.shape == (378, 378, 3) for crop in result["local_crops"])
+    assert len(result["tiling"]) == 2
+
+
+def test_overlap_crop_small_image():
+    # Test with image smaller than crop size
+    test_image = np.zeros((300, 200, 3), dtype=np.uint8)
+    result = overlap_crop_image(test_image, overlap_margin=4, max_crops=12)
+
+    # Should still produce valid output
+    assert result["global_crop"].shape == (378, 378, 3)
+    assert len(result["local_crops"]) == 1
+    assert result["tiling"] == (1, 1)
+
+
+def test_reconstruction():
+    # Create a test image
+    test_image = np.zeros((800, 600, 3), dtype=np.uint8)
+    # Add a recognizable pattern
+    test_image[300:500, 200:400] = 255
+
+    # Crop and reconstruct
+    result = overlap_crop_image(test_image, overlap_margin=4, max_crops=12)
+    crops_tensor = [torch.from_numpy(crop) for crop in result["local_crops"]]
+    reconstructed = reconstruct_from_crops(
+        crops_tensor, result["tiling"], overlap_margin=4
+    )
+
+    # Convert back to numpy for comparison
+    reconstructed_np = reconstructed.numpy()
+
+    # The reconstructed image should be similar to the input
+    # We can't expect exact equality due to resizing operations
+    # but the white rectangle should still be visible in the middle
+    center_original = test_image[300:500, 200:400].mean()
+    center_reconstructed = reconstructed_np[
+        reconstructed_np.shape[0] // 2 - 100 : reconstructed_np.shape[0] // 2 + 100,
+        reconstructed_np.shape[1] // 2 - 100 : reconstructed_np.shape[1] // 2 + 100,
+    ].mean()
+
+    # The center region should be significantly brighter than the edges
+    assert center_reconstructed > reconstructed_np[:100, :100].mean() + 100