Refactor prefill and inference benchmark (#184) #101

	name: CLI CUDA Torch-ORT Multi-GPU Tests

	on:
	workflow_dispatch:
	push:
	branches: [main]
	pull_request:
	types: [opened, reopened, synchronize]

	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: true

	jobs:
	build_image_and_run_cli_cuda_torch_ort_multi_gpu_tests:
	strategy:
	fail-fast: false
	matrix:
	image: [{ cuda_version: 11.8.0, torch_cuda: cu118 }]

	runs-on: [multi-gpu, nvidia-gpu, 4-a10, ci]

	steps:
	- name: Checkout
	uses: actions/checkout@v3

	- name: Build image
	run: docker build
	--build-arg USER_ID=$(id -u)
	--build-arg GROUP_ID=$(id -g)
	--build-arg TORCH_CUDA=${{ matrix.image.torch_cuda }}
	--build-arg CUDA_VERSION=${{ matrix.image.cuda_version }}
	--tag opt-bench-cuda:${{ matrix.image.cuda_version }}
	docker/cuda

	- name: Run tests
	uses: addnab/docker-run-action@v3
	with:
	image: opt-bench-cuda:${{ matrix.image.cuda_version }}
	options: \|
	--rm
	--gpus all
	--shm-size 64G
	--env MKL_THREADING_LAYER=GNU
	--volume ${{ github.workspace }}:/workspace
	--workdir /workspace
	run: \|
	pip install -e .[testing,torch-ort,peft] && python -m torch_ort.configure
	pytest -x -s -k "cli and cuda and torch_ort and (dp or ddp or device_map) and not peft"

Provide feedback