Calculate leaderboard #6
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Auto-Fix Tagged Issues with OpenHands | |
| on: | |
| workflow_call: | |
| inputs: | |
| issue_number: | |
| required: true | |
| type: number | |
| secrets: | |
| LLM_MODELS: | |
| required: true | |
| LLM_API_KEY: | |
| required: true | |
| LLM_BASE_URL: | |
| required: false | |
| PAT_TOKEN: | |
| required: true | |
| PAT_USERNAME: | |
| required: true | |
| issues: | |
| types: [labeled] | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| issues: write | |
| jobs: | |
| auto-fix: | |
| if: github.event_name == 'workflow_call' || github.event.label.name == 'onlycommit' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Check required environment variables | |
| env: | |
| LLM_MODELS: ${{ secrets.LLM_MODELS }} | |
| LLM_API_KEY: ${{ secrets.LLM_API_KEY }} | |
| LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }} | |
| PAT_TOKEN: ${{ secrets.PAT_TOKEN }} | |
| PAT_USERNAME: ${{ secrets.PAT_USERNAME }} | |
| run: | | |
| required_vars=("LLM_MODELS" "LLM_API_KEY" "PAT_TOKEN" "PAT_USERNAME") | |
| for var in "${required_vars[@]}"; do | |
| if [ -z "${!var}" ]; then | |
| echo "Error: Required environment variable $var is not set." | |
| exit 1 | |
| fi | |
| done | |
| - name: Set issue number | |
| run: echo "ISSUE_NUMBER=${{ github.event.issue.number || inputs.issue_number }}" >> $GITHUB_ENV | |
| - name: Comment on issue with start message | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| github.rest.issues.createComment({ | |
| issue_number: ${{ env.ISSUE_NUMBER }}, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: `OpenHands started fixing the issue! You can monitor the progress [here](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}).` | |
| }); | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install git+https://${{ secrets.PAT_TOKEN }}@github.com/neulab/pr-arena.git@main | |
| pip install firebase-admin | |
| - name: Attempt to resolve issues and Send the Resolver Output to firebase. | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| LLM_MODELS: ${{ secrets.LLM_MODELS }} | |
| LLM_API_KEY: ${{ secrets.LLM_API_KEY }} | |
| LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }} | |
| PYTHONPATH: "" | |
| FIREBASE_CONFIG: ${{ secrets.FIREBASE_CONFIG }} | |
| run: | | |
| cd /tmp && python -m openhands_resolver.resolve_issues \ | |
| --repo ${{ github.repository }} \ | |
| --issue-numbers ${{ env.ISSUE_NUMBER }} | |
| - name: Check resolution results | |
| id: check_results | |
| run: | | |
| if cd /tmp && grep -q '"success":true' output/output1.jsonl; then | |
| echo "MODEL1_RESOLUTION_SUCCESS=true" >> $GITHUB_ENV | |
| else | |
| echo "MODEL1_RESOLUTION_SUCCESS=false" >> $GITHUB_ENV | |
| fi | |
| if cd /tmp && grep -q '"success":true' output/output2.jsonl; then | |
| echo "MODEL2_RESOLUTION_SUCCESS=true" >> $GITHUB_ENV | |
| else | |
| echo "MODEL2_RESOLUTION_SUCCESS=false" >> $GITHUB_ENV | |
| fi | |
| # Create commits for both models (if successful) and store their commit hashes in Firebase | |
| - name: Commit changes for model 1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.PAT_TOKEN }} | |
| GITHUB_USERNAME: ${{ secrets.PAT_USERNAME }} | |
| FIREBASE_CONFIG: ${{ secrets.FIREBASE_CONFIG }} | |
| run: | | |
| python commit_only.py \ | |
| --issue-number ${{ env.ISSUE_NUMBER }} \ | |
| --model-number 1 \ | |
| --repo ${{ github.repository }} \ | |
| --github-token ${{ secrets.PAT_TOKEN }} \ | |
| --github-username ${{ secrets.PAT_USERNAME }} \ | |
| --output-dir output \ | |
| --firebase-config "${{ secrets.FIREBASE_CONFIG }}" | |
| - name: Commit changes for model 2 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.PAT_TOKEN }} | |
| GITHUB_USERNAME: ${{ secrets.PAT_USERNAME }} | |
| FIREBASE_CONFIG: ${{ secrets.FIREBASE_CONFIG }} | |
| run: | | |
| python commit_only.py \ | |
| --issue-number ${{ env.ISSUE_NUMBER }} \ | |
| --model-number 2 \ | |
| --repo ${{ github.repository }} \ | |
| --github-token ${{ secrets.PAT_TOKEN }} \ | |
| --github-username ${{ secrets.PAT_USERNAME }} \ | |
| --output-dir output \ | |
| --firebase-config "${{ secrets.FIREBASE_CONFIG }}" | |
| # Now that both commits are created and their hashes stored in Firebase, post the webpage link | |
| - name: Post webpage link to GitHub issue comment | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const issueNumber = ${{ env.ISSUE_NUMBER }}; | |
| const repo = context.repo.repo; | |
| const owner = context.repo.owner; | |
| const firebaseWebpageURL = `https://pr-arena.web.app/${owner}-${repo}-${issueNumber}`; | |
| github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: issueNumber, | |
| body: `⚔️PR-Arena is now open⚔️! You can view the proposed fixes and make a decision at [this link](${firebaseWebpageURL}).` | |
| }); |