From 4cc58afbfcc9eeb8d4ced297be562bbbe063e6ba Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Tue, 5 Aug 2025 13:52:01 +0200 Subject: [PATCH 1/6] configure gh actions --- .github/workflows/python-package.yml | 74 ++++++++++++++++ .github/workflows/python-publish.yml | 31 +++++++ .gitlab-ci.yml | 127 --------------------------- 3 files changed, 105 insertions(+), 127 deletions(-) create mode 100644 .github/workflows/python-package.yml create mode 100644 .github/workflows/python-publish.yml delete mode 100644 .gitlab-ci.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 0000000..c60b71b --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,74 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python package test + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: '0 8 * * 1,4' + +jobs: + unit-tests: + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12"] + os: ["ubuntu-22.04", "ubuntu-latest"] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Lint + run: | + python -m pip install pre-commit + pre-commit run --show-diff-on-failure --all-files + + - name: Install dependencies + env: + CURL_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt + run: | + sudo apt-add-repository -y ppa:ubuntugis/ubuntugis-unstable + sudo apt-get -y update + sudo apt-get install -y gdal-bin libgdal-dev libproj-dev libgeos-dev + python -m pip install --upgrade pip wheel + pip install -e .[complete,test] + pip freeze + + - name: Start containers + run: docker compose -f "docker-compose.yml" up -d + + # run tests + - name: run + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REQUEST_PAYER: ${{ secrets.AWS_REQUEST_PAYER }} + CURL_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt + run: pytest -v --cov mapchete-hub-cli --cov-report xml:coverage.xml --cov-report=term-missing:skip-covered --junitxml=pytest.xml + + # Upload the report to Codecov + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: mapchete/mapchete-hub-cli + + # this will let the workflow fail if coverage is below 100% + - name: Pytest coverage + run: coverage report --skip-covered --show-missing + # --fail-under 100 + + - name: Stop containers + if: always() + run: docker compose -f "docker-compose.yml" down diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 0000000..0499260 --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,31 @@ +# This workflows will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +name: Upload Python Package + +on: + release: + types: [created] + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install hatch + - name: Build and publish + env: + HATCH_INDEX_USER: "__token__" + HATCH_INDEX_AUTH: ${{ secrets.PYPI_TOKEN }} + run: | + hatch build + hatch publish diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 9f10361..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,127 +0,0 @@ -stages: - - codecheck - - build - - test - - package - -# python packaging & EOX PyPy releases # -######################################## - -precommit: - stage: codecheck - tags: - - privileged - image: registry.gitlab.eox.at/maps/docker-base/python:3.11-slim-buster - before_script: - - apt update && apt install -y --no-install-recommends git - - pip install pre-commit - script: - - pre-commit run --all-files - -# build python packages -build_python: - image: docker.io/python:3.10-slim-buster - tags: - - privileged - stage: build - artifacts: - paths: - - dist/ - before_script: - - pip install --upgrade pip hatch - script: - - hatch build - -# test python packages -test_wheel: - image: docker.io/python:3.10-slim-buster - tags: - - privileged - needs: ["build_python"] - stage: test - before_script: - - pip install - --extra-index-url https://__token__:${EOX_PYPI_TOKEN}@gitlab.eox.at/api/v4/projects/255/packages/pypi/simple - -r requirements_test.txt - - pip install - --extra-index-url https://__token__:${EOX_PYPI_TOKEN}@gitlab.eox.at/api/v4/projects/255/packages/pypi/simple - --upgrade - `ls dist/mapchete_hub_cli*.tar.gz -1 | head -1`[test] - script: - - mhub --help - - pytest -v --cov=mapchete_hub_cli tests/ && coverage report - -test_tar: - image: docker.io/python:3.10-slim-buster - tags: - - privileged - needs: ["build_python"] - stage: test - before_script: - - pip install - --extra-index-url https://__token__:${EOX_PYPI_TOKEN}@gitlab.eox.at/api/v4/projects/255/packages/pypi/simple - -r requirements_test.txt - - pip install - --extra-index-url https://__token__:${EOX_PYPI_TOKEN}@gitlab.eox.at/api/v4/projects/255/packages/pypi/simple - --upgrade - `ls dist/mapchete_hub_cli*.tar.gz -1 | head -1`[test] - script: - - mhub --help - - pytest -v --cov=mapchete_hub_cli tests/ && coverage report - - -# INTEGRATION TESTS CURRENTLY NOT WORKING ON OUR CI -# # full test suite # -# test_integration: -# image: docker/compose:1.26.0 -# needs: ["build_python"] -# stage: test -# timeout: 30m -# retry: 2 -# before_script: -# - docker info -# - echo -n $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY -# - export MHUB_PORT=$(( 5000 + $RANDOM % 1000 )) -# script: -# # run tests -# - echo "run mhub on port ${MHUB_PORT}" -# - > -# docker-compose \ -# -p $CI_JOB_ID \ -# -f docker-compose.yml \ -# -f docker-compose.test.yml \ -# up \ -# --exit-code-from mhub_cli_tester \ -# --build -# after_script: -# - > -# docker-compose \ -# -p $CI_JOB_ID \ -# -f docker-compose.yml \ -# -f docker-compose.test.yml \ -# down \ -# -v \ -# --rmi all \ -# --remove-orphans \ -# || true -# - > -# docker-compose \ -# -p $CI_JOB_ID \ -# -f docker-compose.yml \ -# -f docker-compose.test.yml \ -# rm -fv \ -# || true - - -# release package -package: - image: docker.io/python:3.10-slim-buster - tags: - - privileged - stage: package - before_script: - - pip install --upgrade pip hatch - script: - - hatch publish -r https://gitlab.eox.at/api/v4/projects/255/packages/pypi dist/mapchete_hub_cli* - only: - - tags From 7826b6d2c1ffd9eba61534b4a3a08804ff8ccc13 Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Tue, 5 Aug 2025 13:53:12 +0200 Subject: [PATCH 2/6] configure gh actions --- .github/workflows/python-package.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index c60b71b..ce6b0b4 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -53,8 +53,6 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REQUEST_PAYER: ${{ secrets.AWS_REQUEST_PAYER }} - CURL_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt run: pytest -v --cov mapchete-hub-cli --cov-report xml:coverage.xml --cov-report=term-missing:skip-covered --junitxml=pytest.xml # Upload the report to Codecov From a9282dee8b2e2bd0750fb8ae09e924752cba46b8 Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Wed, 6 Aug 2025 11:02:37 +0200 Subject: [PATCH 3/6] use gh image --- docker-compose.test.yml | 21 --------------------- docker-compose.yml | 6 +++--- 2 files changed, 3 insertions(+), 24 deletions(-) delete mode 100644 docker-compose.test.yml diff --git a/docker-compose.test.yml b/docker-compose.test.yml deleted file mode 100644 index 1cc918d..0000000 --- a/docker-compose.test.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: "3.6" - -services: - - mhub_cli_tester: - build: - context: . - args: - BASE_IMAGE: registry.gitlab.eox.at/maps/mapchete_hub/mhub - BASE_IMAGE_TAG: "2024.10.3" - EOX_PYPI_TOKEN: ${EOX_PYPI_TOKEN} - depends_on: - - server - environment: - - MHUB_HOST=http://server:${MHUB_PORT:-5000} - command: /bin/bash -c "echo $MHUB_HOST; sleep 5; pytest -v --cov=mapchete_hub_cli && coverage report" - networks: - - frontend - -networks: - frontend: diff --git a/docker-compose.yml b/docker-compose.yml index 0c82fc0..25b8064 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: command: [ "--bind_ip_all" ] server: - image: registry.gitlab.eox.at/maps/mapchete_hub/mhub:2024.10.3 + image: ghcr.io/mapchete/mapchete-hub:2025.8.0 depends_on: - dask_scheduler - db @@ -41,7 +41,7 @@ services: ] dask_scheduler: - image: registry.gitlab.eox.at/maps/mapchete_hub/mhub:2024.10.3 + image: ghcr.io/mapchete/mapchete-hub:2025.8.0 networks: - backend command: @@ -54,7 +54,7 @@ services: ] dask_worker: - image: registry.gitlab.eox.at/maps/mapchete_hub/mhub:2024.10.3 + image: ghcr.io/mapchete/mapchete-hub:2025.8.0 depends_on: - dask_scheduler networks: From 3ffbbe0eae661c86c244e9e9e852ac55db146e5c Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Wed, 6 Aug 2025 11:18:31 +0200 Subject: [PATCH 4/6] update readme, license & remove requirements files --- LICENSE | 2 +- README.rst | 133 +++++++++++++++++++++++++++++++++++++++--- requirements.txt | 5 -- requirements_dev.txt | 1 - requirements_test.txt | 7 --- 5 files changed, 127 insertions(+), 21 deletions(-) delete mode 100644 requirements.txt delete mode 100644 requirements_dev.txt delete mode 100644 requirements_test.txt diff --git a/LICENSE b/LICENSE index bb9a60b..64946c6 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2018 - 2024 EOX IT Services +Copyright (c) 2018 - 2025 EOX IT Services Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.rst b/README.rst index 3a7be02..9156bc7 100644 --- a/README.rst +++ b/README.rst @@ -1,14 +1,133 @@ -================ mapchete Hub CLI ================ -CLI and Python bindings to mapchete Hub API. +A command line interface for mapchete Hub. + +.. image:: https://img.shields.io/pypi/v/mapchete-hub-cli.svg + :target: https://pypi.org/project/mapchete-hub-cli/ + +.. image:: https://img.shields.io/pypi/l/mapchete-hub-cli.svg + :target: https://github.com/mapchete/mapchete-hub-cli/blob/main/LICENSE + +.. image:: https://img.shields.io/github/actions/workflow/status/mapchete/mapchete-hub-cli/python-package.yml?label=tests + :target: https://github.com/mapchete/mapchete-hub-cli/actions + +.. image:: https://codecov.io/gh/mapchete/mapchete-hub-cli/graph/badge.svg?token=VD1YOF3QA2 + :target: https://codecov.io/gh/mapchete/mapchete-hub-cli + +.. image:: https://img.shields.io/github/repo-size/mapchete/mapchete-hub-cli + :target: https://github.com/mapchete/mapchete-hub-cli + +About mapchete Hub CLI +======================== + +The mapchete Hub CLI provides a user-friendly command line interface (``mhub``) to interact with a running mapchete Hub instance. It simplifies the process of submitting, monitoring, and managing processing jobs without needing to write custom scripts or use tools like `curl`. + +This CLI is the recommended tool for users who want to script workflows or manage mapchete Hub jobs from their terminal. + +--- + +Key Features +============ + +* **💻 Full Job Lifecycle Management**: Intuitive commands to ``execute``, ``job`` (status), ``jobs`` (list), ``cancel``, and ``retry`` processing jobs. +* **⚙️ Hub Inspection**: List available ``processes`` on a remote mapchete Hub and check remote package ``--remote-versions``. +* **🔒 Authentication Support**: Handles basic authentication via command line options (``-u``, ``-p``) or environment variables. +* **🤖 Script-Friendly**: Designed for easy integration into automated scripts and workflows. + +--- + +Installation +============ + +Install the CLI and its dependencies from PyPI: + +.. code-block:: bash + + pip install mapchete-hub-cli + +--- + +Quickstart +========== + +**1. Point to your mapchete Hub** + +All commands require you to specify the mapchete Hub's URL using the ``-h`` or ``--host`` option. If the mapchete Hub is running on the same machine with default settings (``0.0.0.0:5000``), you may not need this option. + +**2. List Available Processes** + +You can see what processes are available on the target mapchete Hub instance: + +.. code-block:: bash + + mhub -h 0.0.0.0:5000 processes -License -------- +**3. Prepare a mapchete File** -MIT License +Create a `mapchete process file `_ using YAML syntax. **Note that all input and output paths must be absolute**, as they are resolved on the remote server. -Copyright (c) 2018 - 2024 `EOX IT Services`_ +For example, `hillshade.mapchete`: + +.. code-block:: yaml + + process: mapchete.processes.examples.hillshade + zoom_levels: + - 10 + pyramid: + grid: geodetic + input: + dem: https://data.example.com/path/to/dem.tif + output: + path: s3://my-bucket/path/to/hillshade_output + format: GTiff + +**4. Execute the Process** + +Use the ``execute`` command to send the job to the mapchete Hub. + +.. code-block:: bash + + mhub -h 0.0.0.0:5000 execute hillshade.mapchete + +The mapchete Hub will return a unique `job_id`: + +.. code-block:: text + + Job submitted with ID: 6227b68b-b7e6-4f40-a396-85763197f481 + +**5. Check Job Status and Progress** + +Use the `job_id` with the ``job`` command to get a detailed status report, or ``progress`` for a quick look at the progress percentage. + +.. code-block:: bash + + # Get detailed status + mhub -h 0.0.0.0:5000 job 6227b68b-b7e6-4f40-a396-85763197f481 + + # Get just the progress + mhub -h 0.0.0.0:5000 progress 6227b68b-b7e6-4f40-a396-85763197f481 + +**6. Manage Jobs** + +You can also list all jobs or cancel a running job: + +.. code-block:: bash + + # List all jobs on the mapchete Hub + mhub -h 0.0.0.0:5000 jobs + + # Cancel a specific job + mhub -h 0.0.0.0:5000 cancel 6227b68b-b7e6-4f40-a396-85763197f481 + +--- + +Contributing +============ + +This is an open-source project and we welcome contributions! Please see the `Contributing Guide `_ in the main ``mapchete`` repository for guidelines on how to get started. + +Acknowledgements +================ -.. _`EOX IT Services`: https://eox.at/ +The initial development of this project was made possible with the resources and support of `EOX IT Services GmbH `_. \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 65098cc..0000000 --- a/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -click -oyaml -pydantic -requests -tqdm \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt deleted file mode 100644 index 3e5a678..0000000 --- a/requirements_dev.txt +++ /dev/null @@ -1 +0,0 @@ -pre_commit \ No newline at end of file diff --git a/requirements_test.txt b/requirements_test.txt deleted file mode 100644 index 001643c..0000000 --- a/requirements_test.txt +++ /dev/null @@ -1,7 +0,0 @@ -httpx -mapchete_hub>=2023.12.0 -pytest -pytest-cov -pytest-env -pytest-mongodb -requests \ No newline at end of file From 7220b714e936b15600cf4b581585e700b1a492ed Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Wed, 6 Aug 2025 11:21:41 +0200 Subject: [PATCH 5/6] remove dashes --- README.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 9156bc7..d3cb919 100644 --- a/README.rst +++ b/README.rst @@ -25,7 +25,6 @@ The mapchete Hub CLI provides a user-friendly command line interface (``mhub``) This CLI is the recommended tool for users who want to script workflows or manage mapchete Hub jobs from their terminal. ---- Key Features ============ @@ -35,7 +34,6 @@ Key Features * **🔒 Authentication Support**: Handles basic authentication via command line options (``-u``, ``-p``) or environment variables. * **🤖 Script-Friendly**: Designed for easy integration into automated scripts and workflows. ---- Installation ============ @@ -46,7 +44,6 @@ Install the CLI and its dependencies from PyPI: pip install mapchete-hub-cli ---- Quickstart ========== @@ -94,7 +91,7 @@ The mapchete Hub will return a unique `job_id`: .. code-block:: text - Job submitted with ID: 6227b68b-b7e6-4f40-a396-85763197f481 + 6227b68b-b7e6-4f40-a396-85763197f481 **5. Check Job Status and Progress** @@ -120,7 +117,6 @@ You can also list all jobs or cancel a running job: # Cancel a specific job mhub -h 0.0.0.0:5000 cancel 6227b68b-b7e6-4f40-a396-85763197f481 ---- Contributing ============ From 703081d1ea806b4868d8588431f0274827fbfe72 Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Wed, 6 Aug 2025 11:22:41 +0200 Subject: [PATCH 6/6] fix typo --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index d3cb919..c618665 100644 --- a/README.rst +++ b/README.rst @@ -30,7 +30,7 @@ Key Features ============ * **💻 Full Job Lifecycle Management**: Intuitive commands to ``execute``, ``job`` (status), ``jobs`` (list), ``cancel``, and ``retry`` processing jobs. -* **⚙️ Hub Inspection**: List available ``processes`` on a remote mapchete Hub and check remote package ``--remote-versions``. +* **⚙️ Inspection**: List available ``processes`` on a remote mapchete Hub and check remote package ``--remote-versions``. * **🔒 Authentication Support**: Handles basic authentication via command line options (``-u``, ``-p``) or environment variables. * **🤖 Script-Friendly**: Designed for easy integration into automated scripts and workflows.