-
Notifications
You must be signed in to change notification settings - Fork 19
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
11 changed files
with
28,105 additions
and
2,593 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
# This workflow will run integration tests for the current project once per day | ||
|
||
name: Evaluation | ||
|
||
on: | ||
schedule: | ||
- cron: "37 14 * * *" # Run at 7:37 AM Pacific Time (14:37 UTC) every day | ||
workflow_dispatch: # Allows triggering the workflow manually in GitHub UI | ||
|
||
# If another scheduled run starts while this workflow is still running, | ||
# cancel the earlier run in favor of the next run. | ||
concurrency: | ||
group: ${{ github.workflow }}-${{ github.ref }} | ||
cancel-in-progress: true | ||
|
||
jobs: | ||
integration-tests: | ||
name: Evaluation | ||
strategy: | ||
matrix: | ||
os: [ubuntu-latest] | ||
python-version: ["3.12"] | ||
runs-on: ${{ matrix.os }} | ||
steps: | ||
- uses: actions/checkout@v4 | ||
- name: Install uv | ||
uses: astral-sh/setup-uv@v4 | ||
- name: Set up Python ${{ matrix.python-version }} | ||
run: uv python install ${{ matrix.python-version }} | ||
- name: Install dependencies | ||
run: | | ||
uv sync --all-extras --dev | ||
uv pip install -U pytest-asyncio vcrpy | ||
- name: Run integration tests | ||
env: | ||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | ||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | ||
LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }} | ||
LANGSMITH_TRACING: true | ||
run: | | ||
make evals |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,34 +1,34 @@ | ||
.PHONY: tests lint format doctest integration_tests_fast evals | ||
|
||
tests: | ||
poetry run python -m pytest --disable-socket --allow-unix-socket -n auto --durations=10 tests/unit_tests | ||
uv run python -m pytest --disable-socket --allow-unix-socket -vv --durations=10 tests/unit_tests | ||
|
||
tests_watch: | ||
poetry run ptw --now . -- -vv -x tests/unit_tests | ||
uv run ptw --now . -- -vv -x tests/unit_tests | ||
|
||
integration_tests: | ||
poetry run python -m pytest -v --durations=10 --cov=trustcall --cov-report=term-missing --cov-report=html --cov-config=.coveragerc tests/integration_tests | ||
uv run python -m pytest -v --durations=10 --cov=trustcall --cov-report=term-missing --cov-report=html --cov-config=.coveragerc tests/integration_tests | ||
|
||
integration_tests_fast: | ||
poetry run python -m pytest -n auto --durations=10 -v --cov=trustcall --cov-report=term-missing --cov-report=html --cov-config=.coveragerc tests/integration_tests | ||
uv run python -m pytest -n auto --durations=10 -v --cov=trustcall --cov-report=term-missing --cov-report=html --cov-config=.coveragerc tests/integration_tests | ||
|
||
evals: | ||
LANGCHAIN_TEST_CACHE=tests/evals/cassettes poetry run python -m pytest -p no:asyncio --max-asyncio-tasks 4 tests/evals/test_evals.py | ||
LANGCHAIN_TEST_CACHE=tests/evals/cassettes uv run python -m pytest tests/evals/test_evals.py | ||
|
||
lint: | ||
poetry run ruff check . | ||
poetry run mypy . | ||
uv run ruff check . | ||
uv run mypy . | ||
|
||
doctest: | ||
poetry run python -m pytest -n auto --durations=10 --doctest-modules trustcall | ||
uv run python -m pytest -n auto --durations=10 --doctest-modules trustcall | ||
|
||
format: | ||
ruff check --select I --fix | ||
poetry run ruff format . | ||
poetry run ruff check . --fix | ||
uv run ruff format . | ||
uv run ruff check . --fix | ||
|
||
build: | ||
poetry build | ||
uv build | ||
|
||
publish: | ||
poetry publish --dry-run | ||
uv publish --dry-run |
Oops, something went wrong.