Split extra libraries and add more granular control over tests

.github/workflows/main.yml

@@ -6,22 +6,24 @@ on:
       - main
   pull_request:
 
+concurrency:
+  # For a given workflow, if we push to the same branch, cancel all previous builds on that branch except on main.
+  group: "${{ github.workflow }}-${{ github.ref }}"
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
 jobs:
   lint:
     name: Linting Suite
     runs-on: ubuntu-latest
     steps:
-      - name: Cancel previous runs
-        uses: styfle/cancel-workflow-action@0.11.0
-        with:
-          access_token: ${{ github.token }}
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
         with:
           python-version: "3.10"
+          cache: 'pip'
       - name: Install tox
         run: |
-          pip install tox>=4.0
+          pip install tox>=4.30.3
       - name: Run linting suite ⚙️
         run: |
           tox -e lint
@@ -32,44 +34,57 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        include:
-          - tox-env: py310-extra
-            python-version: "3.10"
-          - tox-env: py311-extra
-            python-version: "3.11"
-          - tox-env: py312-extra
-            python-version: "3.12"
-          - tox-env: py313-extra
-            python-version: "3.13"
+        python-version: [ "3.10", "3.11", "3.12", "3.13" ]
     steps:
-    - uses: actions/checkout@v4
-    - name: Install packages 📦
-      run: |
-        sudo apt-get update
-        sudo apt-get -y install libnetcdf-dev libhdf5-dev
-    - uses: actions/setup-python@v5
-      name: Setup Python ${{ matrix.python-version }}
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install tox 📦
-      run: pip install "tox>=4.0"
-    - name: Run tests with tox ⚙️
-      run: tox -e ${{ matrix.tox-env }}
-    - name: Run coveralls ⚙️
-      if: matrix.python-version == 3.10
-      uses: AndreMiras/coveralls-python-action@develop
+      - uses: actions/checkout@v6
+      - name: Install packages 📦
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install libnetcdf-dev libhdf5-dev
+      - uses: actions/setup-python@v6
+        name: Setup Python ${{ matrix.python-version }}
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+      - name: Install tox 📦
+        run: pip install "tox>=4.30.3" "tox-gh>=1.5"
+      - name: Run tests with tox ⚙️
+        run: |
+          tox
+        env:
+          TOX_GH_MAJOR_MINOR: ${{ matrix.python-version }}
+      - name: Report Coverage
+        uses: coverallsapp/github-action@v2
+        with:
+          flag-name: run-${{ matrix.python-version }}
+          parallel: true
 
   docs:
     name: Build docs 🏗️
     needs: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
         name: Setup Python 3.10
         with:
           python-version: "3.10"
+          cache: 'pip'
       - name: Build documentation 🏗️
         run: |
           pip install -e .[dev]
           cd docs && make html
+
+  finish:
+    name: Finish
+    needs:
+      - test
+      - docs
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Coveralls Finished
+        uses: coverallsapp/github-action@v2
+        with:
+          parallel-finished: true

.gitignore

@@ -20,3 +20,4 @@ docs/_build
 *.orig
 .coverage
 .pytest_cache
+coverage.lcov

environment.yml

@@ -12,9 +12,12 @@ dependencies:
  - urllib3 >=2.5.0
  - markupsafe >=3.0.3
  - numpy >=1.22.2
- - zarr <3
+ # extras
  - fiona
  - geotiff
+ - netcdf4
+ - tifffile <=2025.5.10
+ - zarr <3.0
  # tests
  - pytest
  - ruff >=0.5.7

pytest.ini

@@ -1,4 +1,11 @@
 [pytest]
-addopts = --import-mode=importlib
+addopts =
+  --import-mode=importlib
+xfail_strict = true
 pythonpath = tests
 testpaths = tests
+markers =
+    online: marks tests requiring network
+    requires_fiona: marks tests requiring fiona module
+    requires_geotiff: marks tests requiring geotiff module
+    requires_netcdf4: marks tests requiring netcdf4 module

requirements-dev.txt

@@ -1,12 +1,11 @@
 bump2version
 coverage
-coveralls
 docutils
 ruff
 pylint
 pytest
 pytest-cov
 sphinx
-tox>=4.0
+tox>=4.30.3
 twine
 wheel

requirements-extra.txt

@@ -1 +1,5 @@
 netCDF4
+fiona
+geotiff
+tifffile<=2025.5.10
+zarr<3.0

requirements.txt

@@ -1,9 +1,5 @@
 markupsafe
 sqlalchemy
-fiona
-geotiff
-tifffile <=2025.5.10
-zarr <3
 humanize
 jinja2
 jsonschema

setup.py

@@ -25,6 +25,9 @@
 with open("requirements-dev.txt") as frd:
     DEV_REQUIRES = frd.read().splitlines()
 
+with open("requirements-extra.txt") as frd:
+    EXTRA_REQUIRES = frd.read().splitlines()
+
 CONFIG = {
     "name": "pywps",
     "version": VERSION,
@@ -58,6 +61,7 @@
     "install_requires": INSTALL_REQUIRES,
     "extras_require": dict(
         dev=DEV_REQUIRES,
+        extra=EXTRA_REQUIRES,
     ),
     "python_requires": ">=3.10,<4",
     "packages": find_packages(exclude=["docs", "tests.*", "tests"]),

tests/__init__.py

@@ -27,7 +27,7 @@
 import test_service
 import test_process
 import test_processing
-import test_assync
+import test_async
 import test_grass_location
 import test_storage
 import test_filestorage
@@ -92,7 +92,7 @@ def load_tests(loader=None, tests=None, pattern=None):
         test_service.load_tests(),
         test_process.load_tests(),
         test_processing.load_tests(),
-        test_assync.load_tests(),
+        test_async.load_tests(),
         test_grass_location.load_tests(),
         test_storage.load_tests(),
         test_filestorage.load_tests(),

tests/test_assync.py → tests/test_async.py

@@ -12,8 +12,6 @@
 from processes import Sleep
 from owslib.wps import WPSExecution
 from pathlib import Path
-from tempfile import TemporaryDirectory
-from pywps import dblog
 
 VERSION = "1.0.0"
 
@@ -27,7 +25,6 @@ def setUp(self) -> None:
         # Running processes using the MultiProcessing scheduler and a file-based database
         configuration.CONFIG.set('processing', 'mode', 'distributed')
 
-    @pytest.mark.xfail(reason="async fails")
     def test_async(self):
         client = client_for(Service(processes=[Sleep()]))
         wps = WPSExecution()
@@ -48,7 +45,6 @@ def test_async(self):
 
         # Parse response to extract the status file path
         url = resp.xml.xpath("//@statusLocation")[0]
-        print(url)
 
         # OWSlib only reads from URLs, not local files. So we need to read the response manually.
         p = Path(configuration.get_config_value('server', 'outputpath')) / url.split('/')[-1]

tests/test_execute.py

@@ -22,12 +22,11 @@
 
 from io import StringIO
 
+netCDF4 = None
 try:
     import netCDF4
 except ImportError:
-    WITH_NC4 = False
-else:
-    WITH_NC4 = True
+    pass
 
 DATA_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data')
 
@@ -235,10 +234,10 @@ def get_output(doc):
 class ExecuteTest(TestBase):
     """Test for Exeucte request KVP request"""
 
-    @pytest.mark.xfail(reason="test.opendap.org is offline")
+    @pytest.mark.online
+    @pytest.mark.requires_netcdf4
+    @pytest.mark.skipif(netCDF4 is None, reason='netCDF4 libraries are required for this test')
     def test_dods(self):
-        if not WITH_NC4:
-            self.skipTest('netCDF4 not installed')
         my_process = create_complex_nc_process()
         service = Service(processes=[my_process])
 
@@ -281,8 +280,11 @@ class FakeRequest():
             language = "en-US"
 
         request = FakeRequest()
-
         resp = service.execute('my_opendap_process', request, 'fakeuuid')
+
+        if resp.outputs["conventions"].data is None:
+            pytest.xfail("Network is likely unavailable or test.opendap.org is offline")
+
         self.assertEqual(resp.outputs['conventions'].data, 'CF-1.0')
         self.assertEqual(resp.outputs['outdods'].url, href)
         self.assertTrue(resp.outputs['outdods'].as_reference)

tests/test_inout.py

@@ -15,6 +15,7 @@
 import json
 from pywps import inout
 import base64
+import pytest
 
 from pywps import Format, FORMATS
 from pywps.app.Common import Metadata
@@ -128,6 +129,7 @@ def test_file(self):
         with self.assertRaises(TypeError):
             self.iohandler[0].data = '5'
 
+    @pytest.mark.online
     def test_url(self):
         if not service_ok('https://demo.mapserver.org'):
             self.skipTest("mapserver is unreachable")
@@ -552,6 +554,7 @@ def test_base64(self):
         b = self.complex_out.base64
         self.assertEqual(base64.b64decode(b).decode(), self.data)
 
+    @pytest.mark.online
     def test_url_handler(self):
         wfsResource = 'http://demo.mapserver.org/cgi-bin/wfs?' \
                       'service=WFS&version=1.1.0&' \
@@ -798,6 +801,7 @@ def test_json(self):
         )
 
 
+@pytest.mark.online
 class TestMetaLink(TestBase):
 
     def setUp(self) -> None:

tests/test_ows.py

@@ -14,6 +14,7 @@
 from pywps import get_ElementMakerForVersion
 import pywps.configuration as config
 from pywps.tests import client_for, assert_response_success, service_ok
+import pytest
 
 wfsResource = 'https://demo.mapserver.org/cgi-bin/wfs?service=WFS&version=1.1.0&request=GetFeature&typename=continents&maxfeatures=10'  # noqa
 wcsResource = 'https://demo.mapserver.org/cgi-bin/wcs?service=WCS&version=1.0.0&request=GetCoverage&coverage=ndvi&crs=EPSG:4326&bbox=-92,42,-85,45&format=image/tiff&width=400&height=300'  # noqa
@@ -91,6 +92,7 @@ def sum_one(request, response):
                            supported_formats=[get_format('GEOTIFF')])],
                        grass_location='epsg:4326')
 
+    @pytest.mark.online
     def test_wfs(self):
         if not service_ok('https://demo.mapserver.org'):
             self.skipTest("mapserver is unreachable")
@@ -117,6 +119,7 @@ def test_wfs(self):
         # . the inclusion of output
         # . the type of output
 
+    @pytest.mark.online
     def test_wcs(self):
         if not config.CONFIG.get('grass', 'gisbase'):
             self.skipTest('GRASS lib not found')

tests/test_s3storage.py

@@ -4,7 +4,7 @@
 ##################################################################
 
 from basic import TestBase
-from pywps.inout.storage.s3 import S3StorageBuilder, S3Storage
+from pywps.inout.storage.s3 import S3StorageBuilder
 from pywps.inout.storage import STORE_TYPE
 from pywps.inout.basic import ComplexOutput
 
@@ -41,7 +41,7 @@ def test_write(self, uploadData):
         configuration.CONFIG.set('s3', 'prefix', 'wps')
         storage = S3StorageBuilder().build()
 
-        url = storage.write('Bar Baz', 'out.txt', data_format=FORMATS.TEXT)
+        storage.write('Bar Baz', 'out.txt', data_format=FORMATS.TEXT)
 
         called_args = uploadData.call_args[0]