diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..4de9727 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,104 @@ +name: Test + +on: + pull_request: + push: + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref_name }} + cancel-in-progress: true + +jobs: + test: + env: + # will be used in the matrix, where neither other variable is used + BOTO_CONFIG: /tmp/nowhere + DATALAD_TESTS_SSH: "1" + DATALAD_LOG_CMD_ENV: GIT_SSH_COMMAND + TESTS_TO_PERFORM: datalad_crawler + PYTEST_OPTS: -s + #PYTEST_SELECTION_OP: "not " # so it would be "not (integration or usecase)" + # Special settings/helper for combined coverage from special remotes execution + COVERAGE: coverage + DATALAD_DATASETS_TOPURL: http://datasets-tests.datalad.org + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: + - '3.7' + - '3.8' + - '3.9' + - '3.10' + - '3.11' + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + # Just in case we need to check if nfs is there etc + - run: sudo lsmod + + - name: Install git-annex + run: | + # The ultimate one-liner setup for NeuroDebian repository + bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh) + sudo apt-get update -qq + sudo apt-get install eatmydata # to speedup some installations + # install git-annex with the relevant bits + # no recommends to avoid inheriting the entire multimedia stack + sudo eatmydata apt-get install --no-install-recommends git-annex-standalone aria2 git-remote-gcrypt lsof gnupg nocache p7zip-full + sudo eatmydata apt-get install zip pandoc + + - name: Install Python dependencies + run: | + # upgrade of importlib-metadata due to https://github.com/pypa/setuptools/issues/3293 + python -m pip install --upgrade pip importlib-metadata + python -m pip install build + python -m pip install codecov + pip install -r requirements-devel.txt + # To heal bloody scrapy -- yoh guesses that pip isn't good enough to + # guarantee specified versioning in the depends + pip install --upgrade attrs + #pip install 'sphinx>=1.6.2' + + - name: Configure Git + run: | + git config --global user.email "test@github.land" + git config --global user.name "GitHub Almighty" + + - name: Verify that building doesn't puke + run: python3 -m build + + - name: Test installation system-wide + run: sudo pip install . + + - name: Run tests + run: | + # -m "$PYTEST_SELECTION_OP(integration or usecase or slow)" + http_proxy= + PATH="$PWD/tools/coverage-bin:$PATH" + pytest $PYTEST_OPTS \ + -v \ + --cov datalad_crawler \ + --cov-report xml \ + --log-cli-level=INFO \ + --pyargs \ + $TESTS_TO_PERFORM + #if [ ! "${DATALAD_LOG_LEVEL:-}" = 2 ]; then + # PYTHONPATH=$PWD make -C docs html doctest + #fi + + - name: Report WTF information using system-wide installed version + run: datalad wtf + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + fail_ci_if_error: false + token: ${{ secrets.CODECOV_TOKEN }} + name: ${{ matrix.python-version }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 710bf5c..0000000 --- a/.travis.yml +++ /dev/null @@ -1,80 +0,0 @@ -# vim ft=yaml -# travis-ci.org definition for DataLad build -language: python -dist: focal - -python: - - 3.7 - - 3.8 - - 3.9 - - 3.10 - - 3.11 - -cache: - - apt - -env: - global: - # will be used in the matrix, where neither other variable is used - - BOTO_CONFIG=/tmp/nowhere - - DATALAD_TESTS_SSH=1 - - DATALAD_LOG_CMD_ENV=GIT_SSH_COMMAND - - TESTS_TO_PERFORM=datalad_crawler - - PYTEST_OPTS=-s - #- PYTEST_SELECTION_OP="not " # so it would be "not (integration or usecase)" - # Special settings/helper for combined coverage from special remotes execution - - COVERAGE=coverage - - DATALAD_DATASETS_TOPURL=http://datasets-tests.datalad.org - -before_install: - # Just in case we need to check if nfs is there etc - - sudo lsmod - # The ultimate one-liner setup for NeuroDebian repository - - bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh) - - travis_retry sudo apt-get update -qq - - travis_retry sudo apt-get install eatmydata # to speedup some installations - # install git-annex with the relevant bits - # no recommends to avoid inheriting the entire multimedia stack - - travis_retry sudo eatmydata apt-get install --no-install-recommends git-annex-standalone aria2 git-remote-gcrypt lsof gnupg nocache p7zip-full - # upgrade of importlib-metadata due to https://github.com/pypa/setuptools/issues/3293 - - pip install --upgrade pip importlib-metadata - -install: - # Install standalone build of git-annex for the recent enough version - - travis_retry sudo eatmydata apt-get install zip pandoc - # for metadata support - - git config --global user.email "test@travis.land" - - git config --global user.name "Travis Almighty" - - cd ..; pip install -q codecov; cd - - - pip install -r requirements-devel.txt - # To heal bloody scrapy -- yoh guesses that pip isn't good enough to guarantee - # specified versioning in the depends - - pip install --upgrade attrs - #- pip install 'sphinx>=1.6.2' - # So we could test under sudo -E with PATH pointing to installed location - - sudo sed -i -e 's/^Defaults.*secure_path.*$//' /etc/sudoers - -script: - # Verify that setup.py build doesn't puke - - python setup.py build - # Test installation system-wide - - sudo pip install . - # Run tests - # -m "$PYTEST_SELECTION_OP(integration or usecase or slow)" - - http_proxy= - PATH=$PWD/tools/coverage-bin:$PATH - pytest $PYTEST_OPTS - -v - --cov datalad_crawler - --log-cli-level=INFO - --pyargs - $TESTS_TO_PERFORM - #- if [ ! "${DATALAD_LOG_LEVEL:-}" = 2 ]; then - # PYTHONPATH=$PWD make -C docs html doctest; - # fi - # Report WTF information using system wide installed version - - datalad wtf - -after_success: - - coverage combine -a /tmp/.coverage-entrypoints-* - - codecov diff --git a/README.md b/README.md index 31ddf91..91bc8a0 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ |____/ \__,_| \__| \__,_||_____| \__,_| \__,_| Crawler -[![Travis tests status](https://secure.travis-ci.org/datalad/datalad-crawler.png?branch=master)](https://travis-ci.org/datalad/datalad-crawler) [![codecov.io](https://codecov.io/github/datalad/datalad-crawler/coverage.svg?branch=master)](https://codecov.io/github/datalad/datalad-crawler?branch=master) [![Documentation](https://readthedocs.org/projects/datalad-crawler/badge/?version=latest)](http://datalad-crawler.rtfd.org) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![GitHub release](https://img.shields.io/github/release/datalad/datalad-crawler.svg)](https://GitHub.com/datalad/datalad-crawler/releases/) [![PyPI version fury.io](https://badge.fury.io/py/datalad-crawler.svg)](https://pypi.python.org/pypi/datalad-crawler/) [![Average time to resolve an issue](http://isitmaintained.com/badge/resolution/datalad/datalad-crawler.svg)](http://isitmaintained.com/project/datalad/datalad-crawler "Average time to resolve an issue") [![Percentage of issues still open](http://isitmaintained.com/badge/open/datalad/datalad-crawler.svg)](http://isitmaintained.com/project/datalad/datalad-crawler "Percentage of issues still open") +[![Test Status](https://github.com/datalad/datalad-crawler/actions/workflows/test.yml/badge.svg)](https://github.com/datalad/datalad-crawler/actions/workflows/test.yml) [![codecov.io](https://codecov.io/github/datalad/datalad-crawler/coverage.svg?branch=master)](https://codecov.io/github/datalad/datalad-crawler?branch=master) [![Documentation](https://readthedocs.org/projects/datalad-crawler/badge/?version=latest)](http://datalad-crawler.rtfd.org) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![GitHub release](https://img.shields.io/github/release/datalad/datalad-crawler.svg)](https://GitHub.com/datalad/datalad-crawler/releases/) [![PyPI version fury.io](https://badge.fury.io/py/datalad-crawler.svg)](https://pypi.python.org/pypi/datalad-crawler/) [![Average time to resolve an issue](http://isitmaintained.com/badge/resolution/datalad/datalad-crawler.svg)](http://isitmaintained.com/project/datalad/datalad-crawler "Average time to resolve an issue") [![Percentage of issues still open](http://isitmaintained.com/badge/open/datalad/datalad-crawler.svg)](http://isitmaintained.com/project/datalad/datalad-crawler "Percentage of issues still open") This extension enhances DataLad (http://datalad.org) for crawling external web resources into an automated data distribution. Please see the [extension