test vllm serially

.github/workflows/test_cli_cuda_vllm.yaml → .github/workflows/test_cli_cuda_vllm_single_gpu.yaml

@@ -50,4 +50,4 @@ jobs:
           run: |
             pip install packaging
             pip install -e .[testing,vllm,flash-attn]
-            pytest -x -s -k "cli and cuda and vllm"
+            FORCE_SERIAL=1 pytest -x -s -k "cli and cuda and vllm"

tests/configs/cuda_inference_vllm_bloom.yaml

@@ -4,6 +4,7 @@ defaults:
   - _cuda_ # inherits from cuda config
   - _inference_ # inherits from inference config
   - _serving_mode_ # inherits from serving_mode config
+  - _no_weights_ # inherits from no weights config
   - _bloom_ # inherits from bloom config
   - _self_ # hydra 1.1 compatibility
   - override backend: vllm

tests/test_cli.py

@@ -8,6 +8,7 @@
 LOGGER = getLogger("test")
 
 
+FORCE_SERIAL = os.environ.get("FORCE_SERIAL", "0") == "1"
 TEST_CONFIG_DIR = "/".join(__file__.split("/")[:-1] + ["configs"])
 TEST_CONFIG_NAMES = [
     config.split(".")[0]
@@ -24,12 +25,17 @@ def test_cli_configs(config_name):
         TEST_CONFIG_DIR,
         "--config-name",
         config_name,
-        # to run the tests faster (comment for debugging)
+        # to run the tests faster
         "hydra/launcher=joblib",
         "hydra.launcher.batch_size=1",
         "hydra.launcher.prefer=threads",
     ]
 
+    if FORCE_SERIAL:
+        args += ["hydra.launcher.n_jobs=1"]
+    else:
+        args += ["hydra.launcher.n_jobs=-1"]
+
     popen = run_subprocess_and_log_stream_output(LOGGER, args)
     assert popen.returncode == 0, f"Failed to run {config_name}"