Call upload model to s3 #38
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Upload model to S3 | |
| on: | |
| workflow_dispatch: | |
| workflow_run: | |
| workflows: ["Model test on push"] | |
| types: | |
| - completed | |
| jobs: | |
| upload_model_to_s3: | |
| if: ${{ github.repository != 'ersilia-os/eos-template' && github.event.workflow_run.conclusion == 'success' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4.2.2 | |
| with: | |
| persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token | |
| fetch-depth: 0 # otherwise, you will failed to push refs to dest repo | |
| lfs: 'true' | |
| - uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| auto-update-conda: true | |
| python-version: "3.12" | |
| auto-activate-base: true | |
| - name: Install dependencies | |
| run: | | |
| conda install git-lfs -c conda-forge | |
| git-lfs install | |
| conda install gh -c conda-forge | |
| python -m pip install git+https://github.com/ersilia-os/ersilia.git | |
| # Here we check if a metadata.yml file exists or a metadata.json file exists | |
| # If metadata.yml file does not exist, we update metadata.json file | |
| - name: Check metadata file | |
| id: checkMetadata | |
| continue-on-error: true | |
| run: | | |
| if [[ ! -f metadata.yml ]]; then | |
| echo "metadata.yml file not found" | |
| exit 1 | |
| fi | |
| - name: Update Metadata JSON file with S3 info | |
| if: steps.checkMetadata.outcome == 'failure' | |
| id: UpdateMetadataJSON | |
| run: | | |
| python3 -c " | |
| import json | |
| with open('metadata.json', 'r') as f: | |
| data = json.load(f) | |
| data['S3'] = 'https://ersilia-models-zipped.s3.eu-central-1.amazonaws.com/{0}.zip'.format(data['Identifier']) | |
| with open('metadata.json', 'w') as f: | |
| json.dump(data, f, indent=4) | |
| " | |
| - name: Update Metadata YAML file with S3 info | |
| if: steps.checkMetadata.outcome == 'success' | |
| id: UpdateMetadataYAML | |
| run: | | |
| python3 -c " | |
| import yaml | |
| with open('metadata.yml', 'r') as f: | |
| data = yaml.safe_load(f) | |
| data['S3'] = 'https://ersilia-models-zipped.s3.eu-central-1.amazonaws.com/{0}.zip'.format(data['Identifier']) | |
| with open('metadata.yml', 'w') as f: | |
| yaml.dump(data, f, default_flow_style=False, sort_keys=False) | |
| " | |
| - name: Commit and push changes done to the Metadata file | |
| uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # pin@v1.4 | |
| with: | |
| author_name: "ersilia-bot" | |
| author_email: "ersilia-bot@users.noreply.github.com" | |
| message: "updating metadata [skip ci]" | |
| repository: "ersilia-os/${{ github.event.repository.name }}" | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| amend: true | |
| force: true | |
| - name: Update metadata to AirTable | |
| id: update-metadata-to-airtable | |
| env: | |
| USER_NAME: ${{ github.repository_owner }} | |
| BRANCH: "main" | |
| REPO_NAME: ${{ github.event.repository.name }} | |
| AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }} | |
| run: | | |
| pip install requests pyairtable | |
| echo "Updating metadata to AirTable looking at owner: $USER_NAME" | |
| wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/airtableops.py | |
| python3 airtableops.py airtable-update --user $USER_NAME --repo $REPO_NAME --branch $BRANCH --api-key $AIRTABLE_API_KEY | |
| rm airtableops.py | |
| - name: sync metadata to S3 JSON | |
| id: sync-metadata-to-s3 | |
| env: | |
| AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| run: | | |
| wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/convert_airtable_to_json.py | |
| pip install boto3 requests pyairtable | |
| python convert_airtable_to_json.py $AIRTABLE_API_KEY $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY | |
| rm convert_airtable_to_json.py | |
| - name: Update README file | |
| id: update-readme-file | |
| env: | |
| MODEL_ID: ${{ github.event.repository.name }} | |
| run: | | |
| echo "Updating README file with AirTable metadata for model: $MODEL_ID" | |
| wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/airtableops.py | |
| python3 airtableops.py readme-update --repo $MODEL_ID --path . | |
| rm airtableops.py | |
| less README.md | |
| - name: Commit and push changes done to the README file | |
| uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # pin@v1.4 | |
| with: | |
| author_name: "ersilia-bot" | |
| author_email: "ersilia-bot@users.noreply.github.com" | |
| message: "updating readme [skip ci]" | |
| repository: "ersilia-os/${{ github.event.repository.name }}" | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| amend: true | |
| force: true | |
| - name: Upload model to S3 | |
| id: upload-model-to-s3 | |
| env: | |
| REPO_NAME: ${{ github.event.repository.name }} | |
| AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| run: | | |
| echo "Uploading model to S3 bucket" | |
| wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/upload_model_to_s3.py | |
| python3 upload_model_to_s3.py $REPO_NAME $AWS_ACCESS_KEY $AWS_SECRET_ACCESS_KEY . | |
| rm upload_model_to_s3.py |