From 6156fdba4f5f5298d38e4bcb9bc47421b48dc7fb Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Tue, 26 Aug 2025 18:00:54 -0400 Subject: [PATCH] run Dockerfile from Python venv and update to Ubuntu Noble --- .github/workflows/containers.yml | 4 +- .github/workflows/docs.yml | 4 +- .github/workflows/flake8.yml | 4 +- .github/workflows/main.yml | 22 +++-- .github/workflows/vulnerabilities.yml | 2 +- .readthedocs.yaml | 4 +- Dockerfile | 37 ++++---- docker/entrypoint.sh | 122 +++++++++++++------------- docs/source/development.rst | 2 +- docs/source/installation.rst | 6 +- docs/source/plugins.rst | 2 +- docs/source/running.rst | 6 +- pyproject.toml | 3 + requirements-provider.txt | 6 +- setup.py | 4 +- 15 files changed, 117 insertions(+), 111 deletions(-) create mode 100644 pyproject.toml diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index f04834da5..ccf300394 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -18,7 +18,7 @@ env: jobs: on-success: name: Build, Test and Push Docker Image to DockerHub - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} permissions: packages: write @@ -80,7 +80,7 @@ jobs: platforms: linux/arm64, linux/amd64 on-failure: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: ${{ github.event.workflow_run.conclusion == 'failure' }} steps: - name: Print Test Fail diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 9de9ab025..e6a0956e7 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -17,11 +17,11 @@ on: jobs: main: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: include: - - python-version: '3.10' + - python-version: '3.12' steps: - uses: actions/checkout@master - uses: actions/setup-python@v5 diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml index b57ba36c2..770c324be 100644 --- a/.github/workflows/flake8.yml +++ b/.github/workflows/flake8.yml @@ -5,13 +5,13 @@ on: jobs: flake8_py3: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@master - uses: actions/setup-python@v5 name: setup Python with: - python-version: '3.10' + python-version: '3.12' - name: Checkout pygeoapi uses: actions/checkout@master - name: Install flake8 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b9429c4ab..b42ccc19d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,11 +17,11 @@ on: jobs: main: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: include: - - python-version: '3.10' + - python-version: '3.12' env: PYGEOAPI_CONFIG: "$(pwd)/pygeoapi-config.yml" @@ -103,18 +103,22 @@ jobs: packages: libsqlite3-mod-spatialite version: 4.3.0a-6build1 - name: Use ubuntuGIS unstable ppa - run: sudo add-apt-repository ppa:ubuntugis/ubuntugis-unstable && sudo apt update + run: | + sudo add-apt-repository ppa:ubuntugis/ubuntugis-unstable + sudo apt update + sudo apt-get install gdal-bin libgdal-dev -y shell: bash - - name: Install GDAL with Python bindings - uses: awalsh128/cache-apt-pkgs-action@v1.4.3 - with: - packages: gdal-bin libgdal-dev - version: 3.8.4 +# - name: Install GDAL with Python bindings +# uses: awalsh128/cache-apt-pkgs-action@v1.4.3 +# with: +# packages: gdal-bin libgdal-dev +# version: 3.11.3 - name: Install and run Oracle run: | docker run -d --name oracledb -e ORACLE_PWD=oracle -v ${{ github.workspace }}/tests/data/oracle/init-db:/opt/oracle/scripts/startup -p 1521:1521 container-registry.oracle.com/database/express:21.3.0-xe - name: Install requirements 📦 run: | + pip3 install setuptools pip3 install -r requirements.txt pip3 install -r requirements-admin.txt pip3 install -r requirements-starlette.txt @@ -122,7 +126,7 @@ jobs: pip3 install -r requirements-provider.txt pip3 install -r requirements-manager.txt pip3 install -r requirements-django.txt - python3 setup.py install + pip3 install . pip3 install --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==`gdal-config --version` - name: setup test data ⚙️ run: | diff --git a/.github/workflows/vulnerabilities.yml b/.github/workflows/vulnerabilities.yml index d8ac5199c..45af41345 100644 --- a/.github/workflows/vulnerabilities.yml +++ b/.github/workflows/vulnerabilities.yml @@ -16,7 +16,7 @@ on: jobs: vulnerabilities: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 defaults: run: working-directory: . diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 350ac42d8..0035b74ea 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -7,9 +7,9 @@ sphinx: configuration: docs/source/conf.py build: - os: ubuntu-22.04 + os: ubuntu-24.04 tools: - python: "3.11" + python: "3.12" python: install: diff --git a/Dockerfile b/Dockerfile index 752f568d2..9809e0e1f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ # Francesco Bartoli # Angelos Tzotsos # -# Copyright (c) 2020 Tom Kralidis +# Copyright (c) 2025 Tom Kralidis # Copyright (c) 2019 Just van den Broecke # Copyright (c) 2025 Francesco Bartoli # Copyright (c) 2025 Angelos Tzotsos @@ -34,7 +34,7 @@ # # ================================================================= -FROM ubuntu:jammy-20250714 +FROM ubuntu:noble-20250805 LABEL maintainer="Just van den Broecke " @@ -93,20 +93,20 @@ ENV TZ=${TZ} \ DEB_PACKAGES="\ locales \ tzdata \ - gunicorn \ python3-dateutil \ python3-gevent \ python3-greenlet \ python3-pip \ python3-tz \ + python3-venv \ python3-yaml \ - ${ADD_DEB_PACKAGES}" + ${ADD_DEB_PACKAGES}" \ + PROJ_LIB=/usr/share/proj WORKDIR /pygeoapi # Install operating system dependencies -RUN \ - apt-get update -y \ +RUN apt-get update -y \ && apt-get install -y ${DEB_BUILD_DEPS} \ && add-apt-repository ppa:ubuntugis/ubuntugis-unstable \ && apt-get --no-install-recommends install -y ${DEB_PACKAGES} \ @@ -126,24 +126,21 @@ RUN \ && apt autoremove -y \ && rm -rf /var/lib/apt/lists/* -ADD requirements-docker.txt requirements-admin.txt /pygeoapi/ -# Install remaining pygeoapi deps -RUN python3 -m pip install --no-cache-dir -r requirements-docker.txt \ - && python3 -m pip install --no-cache-dir -r requirements-admin.txt - - ADD . /pygeoapi - # Install pygeoapi -RUN python3 -m pip install --no-cache-dir -e . +# Install remaining pygeoapi deps and pygeoapi itself +RUN python3 -m venv --system-site-packages /venv \ + && /venv/bin/python3 -m pip install --no-cache-dir -r requirements-docker.txt \ + && /venv/bin/python3 -m pip install --no-cache-dir -r requirements-admin.txt \ + && /venv/bin/python3 -m pip install --no-cache-dir gunicorn \ + && /venv/bin/python3 -m pip install --no-cache-dir -e . -RUN \ - # Set default config and entrypoint for Docker Image - cp /pygeoapi/docker/default.config.yml /pygeoapi/local.config.yml \ +# Set default config and entrypoint for Docker Image +# and compile language files +RUN cp /pygeoapi/docker/default.config.yml /pygeoapi/local.config.yml \ && cp /pygeoapi/docker/entrypoint.sh /entrypoint.sh \ - # compile language files && cd /pygeoapi \ - && for i in locale/*; do echo $i && pybabel compile -d locale -l `basename $i`; done - + && for i in locale/*; do if [ "$i" != "locale/README.md" ]; then echo $i && pybabel compile -d locale -l `basename $i`; fi; done \ + && chmod -R g=u /pygeoapi ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index f84665866..81df84b59 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -3,9 +3,11 @@ # # Authors: Just van den Broecke # Benjamin Webb +# Tom Kralidis # # Copyright (c) 2019 Just van den Broecke # Copyright (c) 2024 Benjamin Webb +# Copyright (c) 2025 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -39,10 +41,10 @@ set +e export PYGEOAPI_HOME=/pygeoapi if [[ -z "$PYGEOAPI_CONFIG" ]]; then - export PYGEOAPI_CONFIG="${PYGEOAPI_HOME}/local.config.yml" + export PYGEOAPI_CONFIG="${PYGEOAPI_HOME}/local.config.yml" fi if [[ -z "$PYGEOAPI_OPENAPI" ]]; then - export PYGEOAPI_OPENAPI="${PYGEOAPI_HOME}/local.openapi.yml" + export PYGEOAPI_OPENAPI="${PYGEOAPI_HOME}/local.openapi.yml" fi # gunicorn env settings with defaults @@ -60,8 +62,8 @@ entry_cmd=${1:-run} # Shorthand function error() { - echo "ERROR: $@" - exit -1 + echo "ERROR: $@" + exit -1 } # Workdir @@ -70,71 +72,71 @@ cd ${PYGEOAPI_HOME} echo "Default config in ${PYGEOAPI_CONFIG}" echo "Trying to generate openapi.yml" -pygeoapi openapi generate ${PYGEOAPI_CONFIG} --output-file ${PYGEOAPI_OPENAPI} +/venv/bin/pygeoapi openapi generate ${PYGEOAPI_CONFIG} --output-file ${PYGEOAPI_OPENAPI} [[ $? -ne 0 ]] && error "openapi.yml could not be generated ERROR" echo "openapi.yml generated continue to pygeoapi" start_gunicorn() { - # SCRIPT_NAME should not have value '/' - [[ "${SCRIPT_NAME}" = '/' ]] && export SCRIPT_NAME="" && echo "make SCRIPT_NAME empty from /" - - echo "Starting gunicorn name=${CONTAINER_NAME} on ${CONTAINER_HOST}:${CONTAINER_PORT} with ${WSGI_WORKERS} workers and SCRIPT_NAME=${SCRIPT_NAME}" - exec gunicorn --workers ${WSGI_WORKERS} \ - --worker-class=${WSGI_WORKER_CLASS} \ - --timeout ${WSGI_WORKER_TIMEOUT} \ - --name=${CONTAINER_NAME} \ - --bind ${CONTAINER_HOST}:${CONTAINER_PORT} \ - ${@} \ - ${WSGI_APP} + # SCRIPT_NAME should not have value '/' + [[ "${SCRIPT_NAME}" = '/' ]] && export SCRIPT_NAME="" && echo "make SCRIPT_NAME empty from /" + + echo "Starting gunicorn name=${CONTAINER_NAME} on ${CONTAINER_HOST}:${CONTAINER_PORT} with ${WSGI_WORKERS} workers and SCRIPT_NAME=${SCRIPT_NAME}" + exec /venv/bin/gunicorn --workers ${WSGI_WORKERS} \ + --worker-class=${WSGI_WORKER_CLASS} \ + --timeout ${WSGI_WORKER_TIMEOUT} \ + --name=${CONTAINER_NAME} \ + --bind ${CONTAINER_HOST}:${CONTAINER_PORT} \ + ${@} \ + ${WSGI_APP} } case ${entry_cmd} in - # Run Unit tests - test) - for test_py in $(ls tests/test_*.py) - do - # Skip tests requiring backend server or libs installed - case ${test_py} in - tests/test_elasticsearch__provider.py) - ;& - tests/test_sensorthings_provider.py) - ;& - tests/test_postgresql_provider.py) - ;& - tests/test_mongo_provider.py) - echo "Skipping: ${test_py}" - ;; - *) - python3 -m pytest ${test_py} - ;; - esac - done - ;; - - # Run pygeoapi server - run) - # Start - start_gunicorn - ;; - - # Run pygeoapi server with hot reload - run-with-hot-reload) - # Lock all Python files (for gunicorn hot reload), if running with user root - if [[ $(id -u) -eq 0 ]] - then - echo "Running pygeoapi as root" - find . -type f -name "*.py" | xargs chmod 0444 - fi - - # Start with hot reload options - start_gunicorn --reload --reload-extra-file ${PYGEOAPI_CONFIG} - ;; - - *) - error "unknown command arg: must be run (default), run-with-hot-reload, or test" - ;; + # Run Unit tests + test) + for test_py in $(ls tests/test_*.py) + do + # Skip tests requiring backend server or libs installed + case ${test_py} in + tests/test_elasticsearch__provider.py) + ;& + tests/test_sensorthings_provider.py) + ;& + tests/test_postgresql_provider.py) + ;& + tests/test_mongo_provider.py) + echo "Skipping: ${test_py}" + ;; + *) + /venv/bin/python3 -m pytest ${test_py} + ;; + esac + done + ;; + + # Run pygeoapi server + run) + # Start + start_gunicorn + ;; + + # Run pygeoapi server with hot reload + run-with-hot-reload) + # Lock all Python files (for gunicorn hot reload), if running with user root + if [[ $(id -u) -eq 0 ]] + then + echo "Running pygeoapi as root" + find . -type f -name "*.py" | xargs chmod 0444 + fi + + # Start with hot reload options + start_gunicorn --reload --reload-extra-file ${PYGEOAPI_CONFIG} + ;; + + *) + error "unknown command arg: must be run (default), run-with-hot-reload, or test" + ;; esac echo "END /entrypoint.sh" diff --git a/docs/source/development.rst b/docs/source/development.rst index 6c8cd1ce4..538dd2a6d 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -134,7 +134,7 @@ Install Python with the option to enable SQLite extensions: .. code-block:: bash - LDFLAGS="-L/usr/local/opt/sqlite/lib -L/usr/local/opt/zlib/lib" CPPFLAGS="-I/usr/local/opt/sqlite/include -I/usr/local/opt/zlib/include" PYTHON_CONFIGURE_OPTS="--enable-loadable-sqlite-extensions" pyenv install 3.10.12 + LDFLAGS="-L/usr/local/opt/sqlite/lib -L/usr/local/opt/zlib/lib" CPPFLAGS="-I/usr/local/opt/sqlite/include -I/usr/local/opt/zlib/include" PYTHON_CONFIGURE_OPTS="--enable-loadable-sqlite-extensions" pyenv install 3.12.3 Configure SQLite from Homebrew over that one shipped with the OS: diff --git a/docs/source/installation.rst b/docs/source/installation.rst index f97c0e6b5..67f982069 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -14,8 +14,8 @@ pygeoapi runs on Python 3. .. note:: The exact Python version requirements are aligned with the version of Python on the pygeoapi supported Ubuntu - operating system version. For example, as of 2024-07, the supported version of Python is bound to Ubuntu 22.04 - (Jammy) which supports Python 3.10. Ensure you have a Python version that is compatible with the current Ubuntu + operating system version. For example, as of 2024-07, the supported version of Python is bound to Ubuntu 24.04 + (Noble) which supports Python 3.12. Ensure you have a Python version that is compatible with the current Ubuntu version that is specified in pygeoapi's `Dockerfile`_. Core dependencies are included as part of a given pygeoapi installation procedure. More specific requirements @@ -34,7 +34,7 @@ For developers and the truly impatient cd pygeoapi pip3 install --upgrade pip pip3 install -r requirements.txt - python3 setup.py install + pip3 install . cp pygeoapi-config.yml example-config.yml vi example-config.yml # edit as required export PYGEOAPI_CONFIG=example-config.yml diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index cadf93a73..848038d87 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -51,7 +51,7 @@ The following methods are options to connect a plugin to pygeoapi: **Option 1**: implement outside of pygeoapi and add to configuration (recommended) * Create a Python package with the plugin code (see `Cookiecutter`_ as an example) -* Install this Python package onto your system (``python3 setup.py install``). At this point your new package +* Install this Python package onto your system (``pip3 install .``). At this point your new package should be in the ``PYTHONPATH`` of your pygeoapi installation * Specify the main plugin class as the ``name`` of the relevant type in the pygeoapi configuration. For example, for a new vector data provider: diff --git a/docs/source/running.rst b/docs/source/running.rst index c92bbee80..ec426c197 100644 --- a/docs/source/running.rst +++ b/docs/source/running.rst @@ -28,7 +28,7 @@ development. To do so, run the following command: .. note:: * Changes to the configuration files of pygeoapi or OpenAPI requires a server restart (configurations are loaded once at server startup for performance). - * Changes to the codebase require a rebuild (i.e., re-running the ``python3 setup.py install`` command). For instructions for running pygeoapi with hot-reloading, refer to the "Hot-reloading" section. + * Changes to the codebase require a rebuild (i.e., re-running the ``pip3 install .`` command). For instructions for running pygeoapi with hot-reloading, refer to the "Hot-reloading" section. Flask WSGI ^^^^^^^^^^ @@ -169,8 +169,8 @@ Hot-reloading ^^^^^^^^^^^^^ The ``pygeoapi serve`` uses the current pygeoapi installation. If the installation was performed using the setup command -provided in the :ref:`install` section (``python3 setup.py install``), changes made to the codebase of pygeoapi are not going to be -reflected in the application until a rebuild (i.e., re-running ``python3 setup.py install``). +provided in the :ref:`install` section (``pip3 install .``), changes made to the codebase of pygeoapi are not going to be +reflected in the application until a rebuild (i.e., re-running ``pip3 install .``). By hot-reloading we mean to be able to directly see changes reflected in the application without reinstalling the pygeoapi package or resetting the server. This is useful for development, as the changes made by developers are easily and rapidly reflected and they can take advantage diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..205a289ec --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=46.4", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/requirements-provider.txt b/requirements-provider.txt index dc70631a0..c22fc5fd1 100644 --- a/requirements-provider.txt +++ b/requirements-provider.txt @@ -4,7 +4,7 @@ cftime elasticsearch elasticsearch-dsl fiona -GDAL<=3.8.4 +GDAL<=3.11.3 geoalchemy2 geopandas netCDF4 @@ -23,5 +23,5 @@ pymysql scipy sodapy xarray -zarr -s3fs<=2023.6.0 \ No newline at end of file +zarr<3 +s3fs<=2023.6.0 diff --git a/setup.py b/setup.py index 7013179aa..161941e21 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2022 Tom Kralidis +# Copyright (c) 2025 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -155,7 +155,7 @@ def get_package_version(): maintainer='Tom Kralidis', maintainer_email='tomkralidis@gmail.com', url='https://pygeoapi.io', - python_requires='>=3.10', + python_requires='>=3.12', install_requires=read('requirements.txt').splitlines(), packages=find_packages(exclude=['pygeoapi.tests']), include_package_data=True,