ci-lint-validate-convert.yml

name: Make JSON-LD and Build Dependencies Table

on:
  pull_request:
    branches: 'main'
    paths: 'HTAN.model.csv'
  release:
    types: [published]
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref_type }}-${{ github.sha }}
  cancel-in-progress: true

env:
  SCHEMATIC_VERSION: '24.2.1'
  GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}

jobs:
  lint:
    name: Lint CSV
    if: always()
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: csvlinter
        uses: kcheriyath/csvlinter@V0.6.0
        with: 
          find_pattern: "*.csv"

  validate:
    name: Validate CSV
    if: always()
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      
      - name: Install IPython
        run: |
          pip install ipython==8.18.1    

      - name: Install Schematic
        shell: bash
        run: |
          pip install schematicpy==$SCHEMATIC_VERSION

      - name: Convert CSV schema
        shell: bash
        run: |
          schematic schema convert .github/CSV.model.csv

      - name: Validate data model CSV
        shell: bash
        run: |
          schematic model -c .github/CSV_schematic_config.yml validate -mp HTAN.model.csv -dt "DataModel" |
            grep "Your manifest has been validated successfully. There are no errors in your manifest, and it can be submitted without any modifications."

  convert:
    name: Convert CSV to JSON-LD
    needs:
      - lint
      - validate
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.10'

      - name: Install IPython
        run: |
          pip install ipython==8.18.1

      - name: Install Schematic
        shell: bash
        run: |
          pip install schematicpy==$SCHEMATIC_VERSION

      - name: Convert .csv to .jsonld
        shell: bash
        run: |
          schematic schema convert HTAN.model.csv

      - uses: r-lib/actions/pr-fetch@v2
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Commit the changes
        run: |
          git add HTAN.model.jsonld
          git commit -m "GitHub Action: convert *.model.csv to *.model.jsonld" || echo "No changes to commit"

      - uses: r-lib/actions/pr-push@v2
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}

  build-dependencies:
    name: Build Dependencies Table
    needs: convert
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v2
        with:
          fetch-depth: 0

      - name: Fetch latest changes
        run: | 
          git fetch origin main
          git merge origin/main

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install requests pandas

      - name: Fetch attribute table
        run: |
          python - <<EOF
          import requests
          import pandas as pd
          from io import StringIO
          
          def fetch_and_save_attribute_table():
              url = "https://schematic.api.sagebionetworks.org/v1/visualize/attributes"
              jsonld_url = "https://raw.githubusercontent.com/ncihtan/data-models/main/HTAN.model.jsonld"
              
              params = {'schema_url': jsonld_url}
              
              print("Fetching attribute table...")
              response = requests.get(url, params=params)
              print(f"Status Code: {response.status_code}")
              
              if response.status_code == 200:
                  content_type = response.headers.get('Content-Type', '')
                  if 'text/html' in content_type:
                      print("Response in text/html format, converting to CSV...")
                      try:
                          df = pd.read_csv(StringIO(response.text))
                          print("Fetched attribute table.")
                          
                          df.to_csv('HTAN.dependencies.csv', index=False)
                          print("Saved to HTAN.dependencies.csv.")
                      except Exception as e:
                          print(f"Error processing CSV: {e}")
                  else:
                      print("Unexpected content type. Expected text/html but got:", content_type)
              else:
                  print(f"Failed to fetch data. Status Code: {response.status_code}")
                  
          fetch_and_save_attribute_table()
          EOF

      - name: Commit changes
        run: |
          git config --global user.name "github-actions[bot]"
          git config --global user.email "github-actions[bot]@users.noreply.github.com"
          git add HTAN.dependencies.csv
          git commit -m "Update HTAN.dependencies.csv with the latest attribute table" || echo "No changes to commit"
  
      - name: Create Pull Request
        uses: peter-evans/create-pull-request@v5
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          commit-message: "Update HTAN.dependencies.csv"
          branch: update-attribute-table
          title: "Update HTAN.dependencies.csv"
          body: "This PR updates HTAN.dependencies.csv with the latest attribute table."