From 632838d67477a709c90ef835094e33a1b3a53e7b Mon Sep 17 00:00:00 2001 From: cre-os Date: Tue, 28 Jan 2025 18:43:56 +0100 Subject: [PATCH 01/11] Unpin dependencies Add monthly tests run to check compatibility with newer dependencies versions. --- .github/workflows/integration-tests-duckdb.yml | 2 ++ .github/workflows/integration-tests-mssql.yml | 2 ++ .github/workflows/integration-tests-mysql.yml | 2 ++ .github/workflows/integration-tests-postgres.yml | 2 ++ .github/workflows/python-package.yml | 9 +++++++-- pyproject.toml | 8 ++++---- 6 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration-tests-duckdb.yml b/.github/workflows/integration-tests-duckdb.yml index 7c63c5b..8492858 100644 --- a/.github/workflows/integration-tests-duckdb.yml +++ b/.github/workflows/integration-tests-duckdb.yml @@ -5,6 +5,8 @@ on: branches: [ "main" ] pull_request: branches: [ "main" ] + schedule: + - cron: "34 3 8 * *" workflow_dispatch: jobs: diff --git a/.github/workflows/integration-tests-mssql.yml b/.github/workflows/integration-tests-mssql.yml index bd7bb87..591e393 100644 --- a/.github/workflows/integration-tests-mssql.yml +++ b/.github/workflows/integration-tests-mssql.yml @@ -5,6 +5,8 @@ on: branches: [ "main" ] pull_request: branches: [ "main" ] + schedule: + - cron: "34 3 8 * *" workflow_dispatch: jobs: diff --git a/.github/workflows/integration-tests-mysql.yml b/.github/workflows/integration-tests-mysql.yml index 2adef2d..0c69cf0 100644 --- a/.github/workflows/integration-tests-mysql.yml +++ b/.github/workflows/integration-tests-mysql.yml @@ -5,6 +5,8 @@ on: branches: [ "main" ] pull_request: branches: [ "main" ] + schedule: + - cron: "34 3 8 * *" workflow_dispatch: jobs: diff --git a/.github/workflows/integration-tests-postgres.yml b/.github/workflows/integration-tests-postgres.yml index b74b2ca..a8ee0fa 100644 --- a/.github/workflows/integration-tests-postgres.yml +++ b/.github/workflows/integration-tests-postgres.yml @@ -5,6 +5,8 @@ on: branches: [ "main" ] pull_request: branches: [ "main" ] + schedule: + - cron: "34 3 8 * *" workflow_dispatch: jobs: diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 4752552..14e3855 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -8,6 +8,8 @@ on: branches: [ "main" ] pull_request: branches: [ "main" ] + schedule: + - cron: "34 3 8 * *" workflow_dispatch: jobs: @@ -17,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 @@ -28,7 +30,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install flake8 .[tests] + python -m pip install flake8 .[tests,docs] - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names @@ -38,3 +40,6 @@ jobs: - name: Test with pytest run: | pytest tests -x -m "not dbtest" + - name: Try to build docs + run: | + mkdocs build --strict diff --git a/pyproject.toml b/pyproject.toml index d89ca70..dec8581 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "xml2db" -version = "0.12.3" +version = "0.12.4" authors = [ { name="Commission de régulation de l'énergie", email="opensource@cre.fr" }, ] @@ -18,12 +18,12 @@ classifiers = [ ] dependencies = [ "sqlalchemy>1.4", - "xmlschema==3.3.2", - "lxml==5.1.0", + "xmlschema>=3.3.2", + "lxml>=5.1.0", ] [project.optional-dependencies] -docs = ["mkdocs-material==9.5.34", "mkdocstrings-python==1.11.1"] +docs = ["mkdocs-material>=9.5.34", "mkdocstrings-python>=1.11.1"] tests = ["pytest>=7.0"] [project.urls] From c82a75837664af5d6f78164b6fa4094a5a8b40d4 Mon Sep 17 00:00:00 2001 From: cre-os Date: Wed, 29 Jan 2025 10:20:52 +0100 Subject: [PATCH 02/11] Add workflow to update requirements.txt --- .../workflows/integration-tests-duckdb.yml | 6 --- .github/workflows/integration-tests-mssql.yml | 6 --- .github/workflows/integration-tests-mysql.yml | 6 --- .../workflows/integration-tests-postgres.yml | 6 --- .github/workflows/python-package.yml | 6 --- .github/workflows/update-requirements.yml | 39 +++++++++++++++++++ 6 files changed, 39 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/update-requirements.yml diff --git a/.github/workflows/integration-tests-duckdb.yml b/.github/workflows/integration-tests-duckdb.yml index 8492858..6d2816a 100644 --- a/.github/workflows/integration-tests-duckdb.yml +++ b/.github/workflows/integration-tests-duckdb.yml @@ -1,12 +1,6 @@ name: Duckdb integration tests on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - schedule: - - cron: "34 3 8 * *" workflow_dispatch: jobs: diff --git a/.github/workflows/integration-tests-mssql.yml b/.github/workflows/integration-tests-mssql.yml index 591e393..f190464 100644 --- a/.github/workflows/integration-tests-mssql.yml +++ b/.github/workflows/integration-tests-mssql.yml @@ -1,12 +1,6 @@ name: MSSQL integration tests on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - schedule: - - cron: "34 3 8 * *" workflow_dispatch: jobs: diff --git a/.github/workflows/integration-tests-mysql.yml b/.github/workflows/integration-tests-mysql.yml index 0c69cf0..3fb4d33 100644 --- a/.github/workflows/integration-tests-mysql.yml +++ b/.github/workflows/integration-tests-mysql.yml @@ -1,12 +1,6 @@ name: MySQL integration tests on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - schedule: - - cron: "34 3 8 * *" workflow_dispatch: jobs: diff --git a/.github/workflows/integration-tests-postgres.yml b/.github/workflows/integration-tests-postgres.yml index a8ee0fa..cdc345f 100644 --- a/.github/workflows/integration-tests-postgres.yml +++ b/.github/workflows/integration-tests-postgres.yml @@ -1,12 +1,6 @@ name: PostgreSQL integration tests on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - schedule: - - cron: "34 3 8 * *" workflow_dispatch: jobs: diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 14e3855..09bb1cd 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -4,12 +4,6 @@ name: Python package on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - schedule: - - cron: "34 3 8 * *" workflow_dispatch: jobs: diff --git a/.github/workflows/update-requirements.yml b/.github/workflows/update-requirements.yml new file mode 100644 index 0000000..6954a22 --- /dev/null +++ b/.github/workflows/update-requirements.yml @@ -0,0 +1,39 @@ +# This workflow will install Python dependencies and create a PR to update requirements.txt if needed +# The goal is to make sure that tests still run with newer dependencies versions, in order to allow +# lax dependencies versions. + +name: Update requirements.txt + +on: + schedule: + - cron: "34 3 8 * *" + workflow_dispatch: + +jobs: + update: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: 3.12 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install .[tests,docs] pyodbc psycopg2 pymysql duckdb_engine + - name: Update requirements.txt + run: | + pip freeze | grep -v 'xml2db' > requirements.txt + - name: Create pull request + if: git diff --quiet --name-only --exit-code requirements.txt && echo false || echo true + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git checkout -b maintenance/update_requirements + git add requirements.txt + git commit -m "update requirements.txt" + git push + echo "ready to do pr" From c8433099550af413688d1d01d14024f2e40e4961 Mon Sep 17 00:00:00 2001 From: cre-os Date: Wed, 29 Jan 2025 10:25:44 +0100 Subject: [PATCH 03/11] trigger workflow --- .github/workflows/update-requirements.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/update-requirements.yml b/.github/workflows/update-requirements.yml index 6954a22..1387517 100644 --- a/.github/workflows/update-requirements.yml +++ b/.github/workflows/update-requirements.yml @@ -5,6 +5,8 @@ name: Update requirements.txt on: + pull_request: + branches: [ "main" ] schedule: - cron: "34 3 8 * *" workflow_dispatch: From 61c44c70b3e9cf674a27d391099fc28563bb0e3a Mon Sep 17 00:00:00 2001 From: cre-os Date: Wed, 29 Jan 2025 10:39:08 +0100 Subject: [PATCH 04/11] update workflow --- .github/workflows/update-requirements.yml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/update-requirements.yml b/.github/workflows/update-requirements.yml index 1387517..08cb9c7 100644 --- a/.github/workflows/update-requirements.yml +++ b/.github/workflows/update-requirements.yml @@ -29,13 +29,16 @@ jobs: - name: Update requirements.txt run: | pip freeze | grep -v 'xml2db' > requirements.txt - - name: Create pull request - if: git diff --quiet --name-only --exit-code requirements.txt && echo false || echo true + - name: Check changes and create pull request run: | - git config user.name github-actions[bot] - git config user.email 41898282+github-actions[bot]@users.noreply.github.com - git checkout -b maintenance/update_requirements - git add requirements.txt - git commit -m "update requirements.txt" - git push - echo "ready to do pr" + if ! git diff --quiet --name-only --exit-code requirements.txt; then + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git checkout -b maintenance/update_requirements + git add requirements.txt + git commit -m "update requirements.txt" + git push + echo "ready to do pr" + else + echo "no changes, nothing to do." + fi From 14e9799f27ac05647e9d1853fdcec7ef31934365 Mon Sep 17 00:00:00 2001 From: cre-os Date: Wed, 29 Jan 2025 10:40:56 +0100 Subject: [PATCH 05/11] update workflow --- .github/workflows/update-requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-requirements.yml b/.github/workflows/update-requirements.yml index 08cb9c7..02367b5 100644 --- a/.github/workflows/update-requirements.yml +++ b/.github/workflows/update-requirements.yml @@ -37,7 +37,7 @@ jobs: git checkout -b maintenance/update_requirements git add requirements.txt git commit -m "update requirements.txt" - git push + git push -u origin maintenance/update_requirements echo "ready to do pr" else echo "no changes, nothing to do." From 2c198e748905e956f47d70d904303ee8ae26d1a1 Mon Sep 17 00:00:00 2001 From: cre-os Date: Wed, 29 Jan 2025 10:43:38 +0100 Subject: [PATCH 06/11] add permission --- .github/workflows/update-requirements.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/update-requirements.yml b/.github/workflows/update-requirements.yml index 02367b5..4f67dd5 100644 --- a/.github/workflows/update-requirements.yml +++ b/.github/workflows/update-requirements.yml @@ -11,6 +11,9 @@ on: - cron: "34 3 8 * *" workflow_dispatch: +permissions: + contents: write + jobs: update: From abb923b94c5ee461ca125ea29f687215cd25ed1b Mon Sep 17 00:00:00 2001 From: cre-os Date: Wed, 29 Jan 2025 10:51:00 +0100 Subject: [PATCH 07/11] test requirements.txt with no changes --- requirements.txt | 55 ++++++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/requirements.txt b/requirements.txt index ee326b6..73b4a7d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,42 +1,47 @@ babel==2.16.0 -certifi==2024.8.30 -charset-normalizer==3.3.2 -click==8.1.7 +certifi==2024.12.14 +charset-normalizer==3.4.1 +click==8.1.8 colorama==0.4.6 -elementpath==4.4.0 +duckdb==1.1.3 +duckdb_engine==0.15.0 +elementpath==4.7.0 ghp-import==2.1.0 -greenlet==3.0.3 -griffe==1.2.0 -idna==3.8 +greenlet==3.1.1 +griffe==1.5.5 +idna==3.10 iniconfig==2.0.0 -Jinja2==3.1.4 -lxml==5.1.0 +Jinja2==3.1.5 +lxml==5.3.0 Markdown==3.7 -MarkupSafe==2.1.5 +MarkupSafe==3.0.2 mergedeep==1.3.4 mkdocs==1.6.1 -mkdocs-autorefs==1.2.0 +mkdocs-autorefs==1.3.0 mkdocs-get-deps==0.2.0 -mkdocs-material==9.5.34 +mkdocs-material==9.5.50 mkdocs-material-extensions==1.3.1 -mkdocstrings==0.26.1 -mkdocstrings-python==1.11.1 -packaging==24.1 +mkdocstrings==0.27.0 +mkdocstrings-python==1.13.0 +packaging==24.2 paginate==0.5.7 pathspec==0.12.1 -platformdirs==4.3.2 +platformdirs==4.3.6 pluggy==1.5.0 -Pygments==2.18.0 -pymdown-extensions==10.9 -pytest==8.3.2 +psycopg2==2.9.10 +Pygments==2.19.1 +pymdown-extensions==10.14.2 +PyMySQL==1.1.1 +pyodbc==5.2.0 +pytest==8.3.4 python-dateutil==2.9.0.post0 PyYAML==6.0.2 pyyaml_env_tag==0.1 -regex==2024.7.24 +regex==2024.11.6 requests==2.32.3 -six==1.16.0 -SQLAlchemy==2.0.34 +six==1.17.0 +SQLAlchemy==2.0.37 typing_extensions==4.12.2 -urllib3==2.2.2 -watchdog==5.0.2 -xmlschema==3.3.2 +urllib3==2.3.0 +watchdog==6.0.0 +xmlschema==3.4.3 From 2a4b966ca2a4f61e848ddd3da17c489f7b043efd Mon Sep 17 00:00:00 2001 From: cre-os Date: Wed, 29 Jan 2025 10:55:44 +0100 Subject: [PATCH 08/11] test create PR --- .github/workflows/update-requirements.yml | 2 +- requirements.txt | 55 +++++++++++------------ 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/.github/workflows/update-requirements.yml b/.github/workflows/update-requirements.yml index 4f67dd5..c100f3a 100644 --- a/.github/workflows/update-requirements.yml +++ b/.github/workflows/update-requirements.yml @@ -41,7 +41,7 @@ jobs: git add requirements.txt git commit -m "update requirements.txt" git push -u origin maintenance/update_requirements - echo "ready to do pr" + gh pr create --base main --head maintenance/update_requirements --title "Update requirements.txt" --body "Update requirements.txt with up to date dependencies versions." else echo "no changes, nothing to do." fi diff --git a/requirements.txt b/requirements.txt index 73b4a7d..ee326b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,47 +1,42 @@ babel==2.16.0 -certifi==2024.12.14 -charset-normalizer==3.4.1 -click==8.1.8 +certifi==2024.8.30 +charset-normalizer==3.3.2 +click==8.1.7 colorama==0.4.6 -duckdb==1.1.3 -duckdb_engine==0.15.0 -elementpath==4.7.0 +elementpath==4.4.0 ghp-import==2.1.0 -greenlet==3.1.1 -griffe==1.5.5 -idna==3.10 +greenlet==3.0.3 +griffe==1.2.0 +idna==3.8 iniconfig==2.0.0 -Jinja2==3.1.5 -lxml==5.3.0 +Jinja2==3.1.4 +lxml==5.1.0 Markdown==3.7 -MarkupSafe==3.0.2 +MarkupSafe==2.1.5 mergedeep==1.3.4 mkdocs==1.6.1 -mkdocs-autorefs==1.3.0 +mkdocs-autorefs==1.2.0 mkdocs-get-deps==0.2.0 -mkdocs-material==9.5.50 +mkdocs-material==9.5.34 mkdocs-material-extensions==1.3.1 -mkdocstrings==0.27.0 -mkdocstrings-python==1.13.0 -packaging==24.2 +mkdocstrings==0.26.1 +mkdocstrings-python==1.11.1 +packaging==24.1 paginate==0.5.7 pathspec==0.12.1 -platformdirs==4.3.6 +platformdirs==4.3.2 pluggy==1.5.0 -psycopg2==2.9.10 -Pygments==2.19.1 -pymdown-extensions==10.14.2 -PyMySQL==1.1.1 -pyodbc==5.2.0 -pytest==8.3.4 +Pygments==2.18.0 +pymdown-extensions==10.9 +pytest==8.3.2 python-dateutil==2.9.0.post0 PyYAML==6.0.2 pyyaml_env_tag==0.1 -regex==2024.11.6 +regex==2024.7.24 requests==2.32.3 -six==1.17.0 -SQLAlchemy==2.0.37 +six==1.16.0 +SQLAlchemy==2.0.34 typing_extensions==4.12.2 -urllib3==2.3.0 -watchdog==6.0.0 -xmlschema==3.4.3 +urllib3==2.2.2 +watchdog==5.0.2 +xmlschema==3.3.2 From 9ce2f370e6977d23104ba31f54b7e0805d7ff7f8 Mon Sep 17 00:00:00 2001 From: cre-os Date: Wed, 29 Jan 2025 11:04:28 +0100 Subject: [PATCH 09/11] setup env --- .github/workflows/update-requirements.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/update-requirements.yml b/.github/workflows/update-requirements.yml index c100f3a..7e8a07b 100644 --- a/.github/workflows/update-requirements.yml +++ b/.github/workflows/update-requirements.yml @@ -45,3 +45,5 @@ jobs: else echo "no changes, nothing to do." fi + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 13dc2ccaa409a328ac9214cf828dd538982b5968 Mon Sep 17 00:00:00 2001 From: cre-os Date: Wed, 29 Jan 2025 11:11:42 +0100 Subject: [PATCH 10/11] update permissions --- .github/workflows/update-requirements.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/update-requirements.yml b/.github/workflows/update-requirements.yml index 7e8a07b..24f536d 100644 --- a/.github/workflows/update-requirements.yml +++ b/.github/workflows/update-requirements.yml @@ -11,12 +11,12 @@ on: - cron: "34 3 8 * *" workflow_dispatch: -permissions: - contents: write - jobs: update: - + permissions: + contents: write + pull-requests: write + repository-projects: write runs-on: ubuntu-latest steps: From 4b14570040f8baf9a7218d64989f99e8108266c2 Mon Sep 17 00:00:00 2001 From: cre-os Date: Wed, 29 Jan 2025 11:24:01 +0100 Subject: [PATCH 11/11] re-establish workflow triggers --- .github/workflows/integration-tests-duckdb.yml | 4 ++++ .github/workflows/integration-tests-mssql.yml | 4 ++++ .github/workflows/integration-tests-mysql.yml | 4 ++++ .github/workflows/integration-tests-postgres.yml | 4 ++++ .github/workflows/python-package.yml | 4 ++++ .github/workflows/update-requirements.yml | 2 -- 6 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests-duckdb.yml b/.github/workflows/integration-tests-duckdb.yml index 6d2816a..7c63c5b 100644 --- a/.github/workflows/integration-tests-duckdb.yml +++ b/.github/workflows/integration-tests-duckdb.yml @@ -1,6 +1,10 @@ name: Duckdb integration tests on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] workflow_dispatch: jobs: diff --git a/.github/workflows/integration-tests-mssql.yml b/.github/workflows/integration-tests-mssql.yml index f190464..bd7bb87 100644 --- a/.github/workflows/integration-tests-mssql.yml +++ b/.github/workflows/integration-tests-mssql.yml @@ -1,6 +1,10 @@ name: MSSQL integration tests on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] workflow_dispatch: jobs: diff --git a/.github/workflows/integration-tests-mysql.yml b/.github/workflows/integration-tests-mysql.yml index 3fb4d33..2adef2d 100644 --- a/.github/workflows/integration-tests-mysql.yml +++ b/.github/workflows/integration-tests-mysql.yml @@ -1,6 +1,10 @@ name: MySQL integration tests on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] workflow_dispatch: jobs: diff --git a/.github/workflows/integration-tests-postgres.yml b/.github/workflows/integration-tests-postgres.yml index cdc345f..b74b2ca 100644 --- a/.github/workflows/integration-tests-postgres.yml +++ b/.github/workflows/integration-tests-postgres.yml @@ -1,6 +1,10 @@ name: PostgreSQL integration tests on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] workflow_dispatch: jobs: diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 09bb1cd..df9bee3 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -4,6 +4,10 @@ name: Python package on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] workflow_dispatch: jobs: diff --git a/.github/workflows/update-requirements.yml b/.github/workflows/update-requirements.yml index 24f536d..796e162 100644 --- a/.github/workflows/update-requirements.yml +++ b/.github/workflows/update-requirements.yml @@ -5,8 +5,6 @@ name: Update requirements.txt on: - pull_request: - branches: [ "main" ] schedule: - cron: "34 3 8 * *" workflow_dispatch: