diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ea82b884..e83640efe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,15 +31,20 @@ All notable changes to this project will be documented in this file. - superset: Add version `4.1.1` ([#991]). - Add Patchable patch management tool ([#1003], [#1005], [#1007]). - nifi: Add 1.28.1, 2.2.0 ([#1006]). -- spark: Add 3.5.5 ([#1022]) +- airflow: Add 2.10.4 ([#1021]). +- hadoop: Add 3.4.1, use jmx_export 1.1.0 ([#1021]). +- hive: Add 4.0.1, use jmx_export 1.1.0 ([#1021]). +- zookeeper: Add 3.9.3, use jmx_export 1.1.0 ([#1021]). +- hbase: Use jmx_export 1.1.0 ([#1021]). +- omid: Use jmx_export 1.1.0 ([#1021]). +- spark: Add 3.5.5 ([#1022]). ### Changed - stackable-base: Bump cargo-cyclonedx to 0.5.7 ([#1013]). -- kafka: Bump 3.8.0 to 3.8.1 ([#995]). - Update registry references to oci ([#989]). - trino-storage-connector: Move the build out of trino/ for easier patching ([#996]). -- BREAKING: druid: Bump opa-authorizer to 0.6.0 for all versions ([#984]) +- BREAKING: druid: Bump opa-authorizer to 0.6.0 for all versions ([#984]). ### Removed @@ -51,7 +56,9 @@ All notable changes to this project will be documented in this file. - trino-storage-connector: Remove version 469 ([#999]). - nifi: Remove 2.0.0 ([#1006]). - druid: Remove 26.0.0 ([#984]) -- spark: Remove 3.5.1 ([#1022]) +- airflow: Remove 2.9.2, and 2.10.2 ([#1021]). +- hive: Remove 4.0.0 ([#1021]). +- spark: Remove 3.5.1 ([#1022]). ### Fixed @@ -60,6 +67,8 @@ All notable changes to this project will be documented in this file. - druid: Fix CVE-2023-34455 in Druid `30.0.0` by deleting a dependency ([#935]). - hadoop: Fix the JMX exporter configuration for metrics suffixed with `_total`, `_info` and `_created` ([#962]). +- nix: Move `pkg-config` to nativeBuildInputs ([#1021]). +- hadoop: Use the fast mirror in the download script ([#1021]). [#884]: https://github.com/stackabletech/docker-images/pull/884 [#928]: https://github.com/stackabletech/docker-images/pull/928 @@ -80,7 +89,6 @@ All notable changes to this project will be documented in this file. [#991]: https://github.com/stackabletech/docker-images/pull/991 [#992]: https://github.com/stackabletech/docker-images/pull/992 [#993]: https://github.com/stackabletech/docker-images/pull/993 -[#995]: https://github.com/stackabletech/docker-images/pull/995 [#996]: https://github.com/stackabletech/docker-images/pull/996 [#997]: https://github.com/stackabletech/docker-images/pull/997 [#999]: https://github.com/stackabletech/docker-images/pull/999 @@ -90,6 +98,7 @@ All notable changes to this project will be documented in this file. [#1006]: https://github.com/stackabletech/docker-images/pull/1006 [#1007]: https://github.com/stackabletech/docker-images/pull/1007 [#1013]: https://github.com/stackabletech/docker-images/pull/1013 +[#1021]: https://github.com/stackabletech/docker-images/pull/1021 [#1022]: https://github.com/stackabletech/docker-images/pull/1022 ## [24.11.1] - 2025-01-14 diff --git a/airflow/constraints-2.10.2-python3.12.txt b/airflow/constraints-2.10.4-python3.12.txt similarity index 55% rename from airflow/constraints-2.10.2-python3.12.txt rename to airflow/constraints-2.10.4-python3.12.txt index 61245fe97..5d94f16ff 100644 --- a/airflow/constraints-2.10.2-python3.12.txt +++ b/airflow/constraints-2.10.4-python3.12.txt @@ -1,6 +1,6 @@ # -# This constraints file was automatically generated on 2024-09-16T16:29:22.054093 +# This constraints file was automatically generated on 2024-12-10T22:43:55.363310 # via "eager-upgrade" mechanism of PIP. For the "v2-10-test" branch of Airflow. # This variant of constraints install uses the HEAD of the branch version for 'apache-airflow' but installs # the providers from PIP-released packages at the moment of the constraint generation. @@ -32,30 +32,29 @@ # Authlib==1.3.2 ConfigUpdater==3.2 -Deprecated==1.2.14 +Deprecated==1.2.15 Events==0.5 -Flask-AppBuilder==4.5.0 +Flask-AppBuilder==4.5.2 Flask-Babel==2.0.0 Flask-Bcrypt==1.0.1 Flask-Caching==2.3.0 -Flask-JWT-Extended==4.6.0 -Flask-Limiter==3.8.0 +Flask-JWT-Extended==4.7.1 +Flask-Limiter==3.9.2 Flask-Login==0.6.3 Flask-SQLAlchemy==2.5.1 Flask-Session==0.5.0 -Flask-WTF==1.2.1 +Flask-WTF==1.2.2 Flask==2.2.5 GitPython==3.1.43 -JPype1==1.5.0 JayDeBeApi==1.2.3 Jinja2==3.1.4 -Mako==1.3.5 +Mako==1.3.8 Markdown==3.7 -MarkupSafe==2.1.5 -PyAthena==3.9.0 -PyGithub==2.4.0 +MarkupSafe==3.0.2 +PyAthena==3.11.0 +PyGithub==2.5.0 PyHive==0.7.0 -PyJWT==2.9.0 +PyJWT==2.10.1 PyNaCl==1.5.0 PyYAML==6.0.2 Pygments==2.18.0 @@ -64,372 +63,378 @@ SQLAlchemy-Utils==0.41.2 SQLAlchemy==1.4.54 SecretStorage==3.3.3 Sphinx==5.3.0 -WTForms==3.1.2 +WTForms==3.2.1 Werkzeug==2.2.3 adal==1.2.7 adlfs==2024.7.0 -aiobotocore==2.15.0 +aiobotocore==2.15.2 aiofiles==23.2.1 -aiohappyeyeballs==2.4.0 -aiohttp==3.10.5 +aiohappyeyeballs==2.4.4 +aiohttp==3.10.11 aioitertools==0.12.0 -aioresponses==0.7.6 +aioresponses==0.7.7 aiosignal==1.3.1 +airbyte-api==0.52.1 alabaster==0.7.16 -alembic==1.13.2 +alembic==1.14.0 alibabacloud-adb20211201==2.0.0 -alibabacloud-tea==0.3.9 -alibabacloud_credentials==0.3.5 +alibabacloud-tea==0.4.0 +alibabacloud_credentials==0.3.6 alibabacloud_endpoint_util==0.0.3 alibabacloud_gateway_spi==0.0.2 alibabacloud_openapi_util==0.2.2 -alibabacloud_tea_openapi==0.3.11 +alibabacloud_tea_openapi==0.3.12 alibabacloud_tea_util==0.3.13 alibabacloud_tea_xml==0.0.2 -aliyun-python-sdk-core==2.15.2 +aliyun-python-sdk-core==2.16.0 aliyun-python-sdk-kms==2.16.5 -amqp==5.2.0 +amqp==5.3.1 analytics-python==1.2.9 annotated-types==0.7.0 anyascii==0.3.2 -anyio==4.4.0 -apache-airflow-providers-airbyte==3.9.0 -apache-airflow-providers-alibaba==2.9.0 -apache-airflow-providers-amazon==8.28.0 +anyio==4.7.0 +apache-airflow-providers-airbyte==4.0.0 +apache-airflow-providers-alibaba==2.9.1 +apache-airflow-providers-amazon==9.1.0 apache-airflow-providers-apache-cassandra==3.6.0 -apache-airflow-providers-apache-drill==2.8.0 -apache-airflow-providers-apache-druid==3.11.0 -apache-airflow-providers-apache-flink==1.5.0 -apache-airflow-providers-apache-hdfs==4.5.0 -apache-airflow-providers-apache-hive==8.2.0 +apache-airflow-providers-apache-drill==2.8.1 +apache-airflow-providers-apache-druid==3.12.1 +apache-airflow-providers-apache-flink==1.5.1 +apache-airflow-providers-apache-hdfs==4.6.0 +apache-airflow-providers-apache-hive==8.2.1 apache-airflow-providers-apache-iceberg==1.1.0 -apache-airflow-providers-apache-impala==1.5.0 -apache-airflow-providers-apache-kafka==1.6.0 +apache-airflow-providers-apache-impala==1.5.2 +apache-airflow-providers-apache-kafka==1.6.1 apache-airflow-providers-apache-kylin==3.7.0 -apache-airflow-providers-apache-livy==3.9.0 +apache-airflow-providers-apache-livy==3.9.2 apache-airflow-providers-apache-pig==4.5.0 -apache-airflow-providers-apache-pinot==4.5.0 -apache-airflow-providers-apache-spark==4.10.0 -apache-airflow-providers-apprise==1.4.0 +apache-airflow-providers-apache-pinot==4.5.1 +apache-airflow-providers-apache-spark==4.11.3 +apache-airflow-providers-apprise==1.4.1 apache-airflow-providers-arangodb==2.6.0 apache-airflow-providers-asana==2.6.0 -apache-airflow-providers-atlassian-jira==2.7.0 -apache-airflow-providers-celery==3.8.1 -apache-airflow-providers-cloudant==3.6.0 -apache-airflow-providers-cncf-kubernetes==8.4.1 +apache-airflow-providers-atlassian-jira==2.7.1 +apache-airflow-providers-celery==3.8.5 +apache-airflow-providers-cloudant==4.0.3 +apache-airflow-providers-cncf-kubernetes==10.0.1 apache-airflow-providers-cohere==1.3.0 -apache-airflow-providers-common-compat==1.2.0 -apache-airflow-providers-common-io==1.4.0 -apache-airflow-providers-common-sql==1.16.0 -apache-airflow-providers-databricks==6.9.0 -apache-airflow-providers-datadog==3.7.0 -apache-airflow-providers-dbt-cloud==3.10.0 +apache-airflow-providers-common-compat==1.2.2 +apache-airflow-providers-common-io==1.4.2 +apache-airflow-providers-common-sql==1.20.0 +apache-airflow-providers-databricks==6.13.0 +apache-airflow-providers-datadog==3.7.1 +apache-airflow-providers-dbt-cloud==3.11.2 apache-airflow-providers-dingding==3.6.0 apache-airflow-providers-discord==3.8.0 -apache-airflow-providers-docker==3.13.0 -apache-airflow-providers-elasticsearch==5.5.0 -apache-airflow-providers-exasol==4.6.0 -apache-airflow-providers-fab==1.3.0 +apache-airflow-providers-docker==3.14.1 +apache-airflow-providers-elasticsearch==5.5.3 +apache-airflow-providers-exasol==4.6.1 +apache-airflow-providers-fab==1.5.1 apache-airflow-providers-facebook==3.6.0 -apache-airflow-providers-ftp==3.11.0 +apache-airflow-providers-ftp==3.11.1 apache-airflow-providers-github==2.7.0 -apache-airflow-providers-google==10.22.0 +apache-airflow-providers-google==11.0.0 apache-airflow-providers-grpc==3.6.0 apache-airflow-providers-hashicorp==3.8.0 -apache-airflow-providers-http==4.13.0 +apache-airflow-providers-http==4.13.3 apache-airflow-providers-imap==3.7.0 -apache-airflow-providers-influxdb==2.7.0 -apache-airflow-providers-jdbc==4.5.0 -apache-airflow-providers-jenkins==3.7.0 -apache-airflow-providers-microsoft-azure==10.4.0 -apache-airflow-providers-microsoft-mssql==3.9.0 +apache-airflow-providers-influxdb==2.7.1 +apache-airflow-providers-jdbc==4.5.3 +apache-airflow-providers-jenkins==3.7.2 +apache-airflow-providers-microsoft-azure==11.1.0 +apache-airflow-providers-microsoft-mssql==3.9.2 apache-airflow-providers-microsoft-psrp==2.8.0 -apache-airflow-providers-microsoft-winrm==3.6.0 -apache-airflow-providers-mongo==4.2.0 -apache-airflow-providers-mysql==5.7.0 +apache-airflow-providers-microsoft-winrm==3.6.1 +apache-airflow-providers-mongo==4.2.2 +apache-airflow-providers-mysql==5.7.4 apache-airflow-providers-neo4j==3.7.0 -apache-airflow-providers-odbc==4.7.0 -apache-airflow-providers-openai==1.3.0 +apache-airflow-providers-odbc==4.8.1 +apache-airflow-providers-openai==1.4.0 apache-airflow-providers-openfaas==3.6.0 -apache-airflow-providers-openlineage==1.11.0 -apache-airflow-providers-opensearch==1.4.0 +apache-airflow-providers-openlineage==1.14.0 +apache-airflow-providers-opensearch==1.5.0 apache-airflow-providers-opsgenie==5.7.0 -apache-airflow-providers-oracle==3.11.0 -apache-airflow-providers-pagerduty==3.8.0 +apache-airflow-providers-oracle==3.12.1 +apache-airflow-providers-pagerduty==3.8.1 apache-airflow-providers-pgvector==1.3.0 -apache-airflow-providers-pinecone==2.1.0 -apache-airflow-providers-postgres==5.12.0 -apache-airflow-providers-presto==5.6.0 +apache-airflow-providers-pinecone==2.1.1 +apache-airflow-providers-postgres==5.14.0 +apache-airflow-providers-presto==5.7.0 apache-airflow-providers-qdrant==1.2.0 apache-airflow-providers-redis==3.8.0 apache-airflow-providers-salesforce==5.8.0 apache-airflow-providers-samba==4.8.0 apache-airflow-providers-segment==3.6.0 apache-airflow-providers-sendgrid==3.6.0 -apache-airflow-providers-sftp==4.11.0 +apache-airflow-providers-sftp==4.11.1 apache-airflow-providers-singularity==3.6.0 -apache-airflow-providers-slack==8.9.0 -apache-airflow-providers-smtp==1.8.0 -apache-airflow-providers-snowflake==5.7.0 -apache-airflow-providers-sqlite==3.9.0 -apache-airflow-providers-ssh==3.13.1 -apache-airflow-providers-tableau==4.6.0 +apache-airflow-providers-slack==8.9.2 +apache-airflow-providers-smtp==1.8.1 +apache-airflow-providers-snowflake==5.8.1 +apache-airflow-providers-sqlite==3.9.1 +apache-airflow-providers-ssh==3.14.0 +apache-airflow-providers-tableau==4.6.1 apache-airflow-providers-tabular==1.6.1 apache-airflow-providers-telegram==4.6.0 -apache-airflow-providers-teradata==2.6.0 -apache-airflow-providers-trino==5.8.0 -apache-airflow-providers-vertica==3.9.0 +apache-airflow-providers-teradata==2.6.1 +apache-airflow-providers-trino==5.9.0 +apache-airflow-providers-vertica==3.9.1 apache-airflow-providers-weaviate==2.1.0 apache-airflow-providers-yandex==3.12.0 -apache-airflow-providers-ydb==1.3.0 +apache-airflow-providers-ydb==2.0.0 apache-airflow-providers-zendesk==4.8.0 -apispec==6.6.1 +apispec==6.8.0 apprise==1.9.0 -argcomplete==3.5.0 +argcomplete==3.5.2 asana==3.2.3 asgiref==3.8.1 asn1crypto==1.5.1 astroid==2.15.8 -asttokens==2.4.1 -asyncssh==2.17.0 +asttokens==3.0.0 +asyncssh==2.18.0 atlasclient==1.0.0 atlassian-python-api==3.41.16 attrs==24.2.0 -aws-sam-translator==1.91.0 +aws-sam-translator==1.94.0 aws-xray-sdk==2.14.0 azure-batch==14.2.0 azure-common==1.1.28 -azure-core==1.31.0 -azure-cosmos==4.7.0 +azure-core==1.32.0 +azure-cosmos==4.9.0 azure-datalake-store==0.0.53 -azure-identity==1.17.1 -azure-keyvault-secrets==4.8.0 -azure-kusto-data==4.5.1 +azure-identity==1.19.0 +azure-keyvault-secrets==4.9.0 +azure-kusto-data==4.6.1 azure-mgmt-containerinstance==10.1.0 azure-mgmt-containerregistry==10.3.0 -azure-mgmt-core==1.4.0 -azure-mgmt-cosmosdb==9.5.1 +azure-mgmt-core==1.5.0 +azure-mgmt-cosmosdb==9.7.0 azure-mgmt-datafactory==9.0.0 azure-mgmt-datalake-nspkg==3.0.1 azure-mgmt-datalake-store==0.5.0 azure-mgmt-nspkg==3.0.2 -azure-mgmt-resource==23.1.1 +azure-mgmt-resource==23.2.0 azure-mgmt-storage==21.2.1 azure-nspkg==3.0.2 -azure-servicebus==7.12.2 -azure-storage-blob==12.22.0 -azure-storage-file-datalake==12.16.0 -azure-storage-file-share==12.17.0 +azure-servicebus==7.13.0 +azure-storage-blob==12.24.0 +azure-storage-file-datalake==12.18.0 +azure-storage-file-share==12.20.0 azure-synapse-artifacts==0.19.0 azure-synapse-spark==0.7.0 babel==2.16.0 backoff==2.2.1 -bcrypt==4.2.0 +bcrypt==4.2.1 beautifulsoup4==4.12.3 -billiard==4.2.0 -bitarray==2.9.2 -black==24.8.0 -blinker==1.8.2 -boto3==1.35.16 -botocore==1.35.16 +billiard==4.2.1 +bitarray==3.0.0 +black==24.10.0 +blinker==1.9.0 +boto3==1.35.36 +botocore==1.35.36 cachelib==0.9.0 cachetools==5.5.0 cassandra-driver==3.29.2 -cattrs==24.1.1 +cattrs==24.1.2 celery==5.4.0 certifi==2024.8.30 cffi==1.17.1 cfgv==3.4.0 -cfn-lint==1.13.0 +cfn-lint==1.22.0 cgroupspy==0.2.3 chardet==5.2.0 -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 checksumdir==1.2.0 -ciso8601==2.3.1 +ciso8601==2.3.2 click-didyoumean==0.3.1 click-plugins==1.1.1 click-repl==0.3.0 click==8.1.7 clickclick==20.10.2 cloudant==2.15.0 -cloudpickle==3.0.0 +cloudpickle==3.1.0 cohere==4.57 colorama==0.4.6 -colorlog==6.8.2 -confluent-kafka==2.5.3 +colorlog==6.9.0 +confluent-kafka==2.6.1 connexion==2.14.2 -coverage==7.6.1 +coverage==7.6.9 crcmod==1.7 cron-descriptor==1.4.5 -croniter==3.0.3 +croniter==5.0.1 cryptography==42.0.8 curlify==2.2.1 databricks-sql-connector==2.9.6 -datadog==0.50.0 -db-dtypes==1.3.0 +dataclasses-json==0.6.7 +datadog==0.50.2 +db-dtypes==1.3.1 decorator==5.1.1 defusedxml==0.7.1 -deltalake==0.19.2 -diagrams==0.23.4 -dill==0.3.8 -distlib==0.3.8 +deltalake==0.22.3 +diagrams==0.24.1 +dill==0.3.9 +distlib==0.3.9 distro==1.9.0 -dnspython==2.6.1 +dnspython==2.7.0 docker==7.1.0 docopt==0.6.2 docstring_parser==0.16 docutils==0.16 -duckdb==1.1.0 -elastic-transport==8.15.0 -elasticsearch==8.15.1 +duckdb==1.1.3 +elastic-transport==8.15.1 +elasticsearch==8.16.0 email_validator==2.2.0 eralchemy2==1.4.1 -et-xmlfile==1.1.0 -eventlet==0.37.0 +et_xmlfile==2.0.0 +eventlet==0.38.1 execnet==2.1.1 executing==2.1.0 -facebook_business==20.0.2 +facebook_business==21.0.5 fastavro==1.9.7 -filelock==3.16.0 +filelock==3.16.1 flower==2.0.1 -frozenlist==1.4.1 -fsspec==2024.9.0 +frozenlist==1.5.0 +fsspec==2024.10.0 future==1.0.0 gcloud-aio-auth==5.3.2 gcloud-aio-bigquery==7.1.0 gcloud-aio-storage==9.3.0 -gcsfs==2024.9.0.post1 +gcsfs==2024.10.0 geomet==0.2.1.post1 -gevent==24.2.1 +gevent==24.11.1 gitdb==4.0.11 -google-ads==25.0.0 -google-analytics-admin==0.23.0 -google-api-core==2.19.2 -google-api-python-client==2.145.0 +google-ads==25.1.0 +google-analytics-admin==0.23.2 +google-api-core==2.24.0 +google-api-python-client==2.154.0 google-auth-httplib2==0.2.0 google-auth-oauthlib==1.2.1 -google-auth==2.34.0 -google-cloud-aiplatform==1.66.0 -google-cloud-appengine-logging==1.4.5 +google-auth==2.36.0 +google-cloud-aiplatform==1.74.0 +google-cloud-appengine-logging==1.5.0 google-cloud-audit-log==0.3.0 -google-cloud-automl==2.13.5 -google-cloud-batch==0.17.27 -google-cloud-bigquery-datatransfer==3.15.7 +google-cloud-automl==2.14.1 +google-cloud-batch==0.17.31 +google-cloud-bigquery-datatransfer==3.17.1 google-cloud-bigquery==3.20.1 -google-cloud-bigtable==2.26.0 -google-cloud-build==3.24.2 -google-cloud-compute==1.19.2 -google-cloud-container==2.51.0 +google-cloud-bigtable==2.27.0 +google-cloud-build==3.27.1 +google-cloud-compute==1.22.0 +google-cloud-container==2.54.0 google-cloud-core==2.4.1 -google-cloud-datacatalog==3.20.1 -google-cloud-dataflow-client==0.8.12 -google-cloud-dataform==0.5.11 -google-cloud-dataplex==2.2.2 -google-cloud-dataproc-metastore==1.15.5 -google-cloud-dataproc==5.11.0 -google-cloud-dlp==3.22.0 -google-cloud-kms==2.24.2 -google-cloud-language==2.14.0 -google-cloud-logging==3.11.2 -google-cloud-memcache==1.9.5 -google-cloud-monitoring==2.22.2 -google-cloud-orchestration-airflow==1.13.1 -google-cloud-os-login==2.14.6 -google-cloud-pubsub==2.23.1 -google-cloud-redis==2.15.5 -google-cloud-resource-manager==1.12.5 -google-cloud-run==0.10.8 -google-cloud-secret-manager==2.20.2 -google-cloud-spanner==3.49.1 -google-cloud-speech==2.27.0 -google-cloud-storage-transfer==1.12.0 -google-cloud-storage==2.18.2 -google-cloud-tasks==2.16.5 -google-cloud-texttospeech==2.17.2 -google-cloud-translate==3.16.0 -google-cloud-videointelligence==2.13.5 -google-cloud-vision==3.7.4 -google-cloud-workflows==1.14.5 +google-cloud-datacatalog==3.23.0 +google-cloud-dataflow-client==0.8.14 +google-cloud-dataform==0.5.13 +google-cloud-dataplex==2.4.0 +google-cloud-dataproc-metastore==1.16.1 +google-cloud-dataproc==5.15.1 +google-cloud-dlp==3.25.1 +google-cloud-kms==3.1.1 +google-cloud-language==2.15.1 +google-cloud-logging==3.11.3 +google-cloud-memcache==1.10.1 +google-cloud-monitoring==2.23.1 +google-cloud-orchestration-airflow==1.15.1 +google-cloud-os-login==2.15.1 +google-cloud-pubsub==2.27.1 +google-cloud-redis==2.16.1 +google-cloud-resource-manager==1.13.1 +google-cloud-run==0.10.13 +google-cloud-secret-manager==2.21.1 +google-cloud-spanner==3.51.0 +google-cloud-speech==2.28.1 +google-cloud-storage-transfer==1.13.1 +google-cloud-storage==2.19.0 +google-cloud-tasks==2.17.1 +google-cloud-texttospeech==2.21.1 +google-cloud-translate==3.18.0 +google-cloud-videointelligence==2.14.1 +google-cloud-vision==3.8.1 +google-cloud-workflows==1.15.1 google-crc32c==1.6.0 google-re2==1.1.20240702 google-resumable-media==2.7.2 -googleapis-common-protos==1.65.0 -graphql-core==3.2.4 +googleapis-common-protos==1.66.0 +graphql-core==3.2.5 graphviz==0.20.3 -greenlet==3.1.0 +greenlet==3.1.1 grpc-google-iam-v1==0.13.1 grpc-interceptor==0.15.4 grpcio-gcp==0.2.2 grpcio-health-checking==1.62.3 grpcio-status==1.62.3 grpcio-tools==1.62.3 -grpcio==1.66.1 -gssapi==1.8.3 +grpcio==1.68.1 +gssapi==1.9.0 gunicorn==23.0.0 h11==0.14.0 h2==4.1.0 -hatch==1.12.0 -hatchling==1.25.0 +hatch==1.13.0 +hatchling==1.26.3 hdfs==2.7.3 hmsclient==0.1.1 hpack==4.0.0 -httpcore==1.0.5 +httpcore==1.0.7 httplib2==0.22.0 httpx==0.27.0 -humanize==4.10.0 +humanize==4.11.0 hvac==2.3.0 hyperframe==6.0.1 hyperlink==21.0.0 +ibm-cloud-sdk-core==3.20.3 +ibmcloudant==0.9.1 icdiff==2.0.7 -identify==2.6.1 +identify==2.6.3 idna==3.10 ijson==3.3.0 imagesize==1.4.1 +immutabledict==4.2.1 importlib-metadata==6.11.0 -importlib_resources==6.4.5 impyla==0.19.0 incremental==24.7.2 inflection==0.5.1 -influxdb-client==1.46.0 +influxdb-client==1.48.0 iniconfig==2.0.0 ipdb==0.13.13 -ipython==8.27.0 -isodate==0.6.1 +ipython==8.30.0 +isodate==0.7.2 itsdangerous==2.2.0 jaraco.classes==3.4.0 jaraco.context==6.0.1 -jaraco.functools==4.0.2 -jedi==0.19.1 +jaraco.functools==4.1.0 +jedi==0.19.2 jeepney==0.8.0 -jiter==0.5.0 +jiter==0.8.2 jmespath==0.10.0 -joserfc==1.0.0 +joserfc==1.0.1 +jpype1==1.5.1 json-merge-patch==0.2 jsondiff==2.2.1 jsonpatch==1.33 -jsonpath-ng==1.6.1 +jsonpath-ng==1.7.0 +jsonpath-python==1.0.6 jsonpointer==3.0.0 jsonschema-path==0.3.3 jsonschema-specifications==2023.12.1 jsonschema==4.23.0 -keyring==25.3.0 -kombu==5.4.1 -krb5==0.6.0 +keyring==25.5.0 +kombu==5.4.2 +krb5==0.7.0 kubernetes==30.1.0 kubernetes_asyncio==30.1.0 kylinpy==2.8.4 lazy-object-proxy==1.10.0 ldap3==2.9.1 -limits==3.13.0 +limits==3.14.1 linkify-it-py==2.0.3 lockfile==0.12.2 -looker-sdk==24.16.1 +looker-sdk==24.20.0 lxml==5.3.0 lz4==4.3.3 markdown-it-py==3.0.0 marshmallow-oneofschema==3.1.1 marshmallow-sqlalchemy==0.28.2 -marshmallow==3.22.0 +marshmallow==3.23.1 matplotlib-inline==0.1.7 mdit-py-plugins==0.4.2 mdurl==0.1.2 @@ -438,40 +443,42 @@ methodtools==0.4.7 microsoft-kiota-abstractions==1.3.3 microsoft-kiota-authentication-azure==1.1.0 microsoft-kiota-http==1.3.3 -mmh3==4.1.0 -mongomock==4.2.0.post1 +microsoft-kiota-serialization-json==1.0.0 +microsoft-kiota-serialization-text==1.0.0 +mmh3==5.0.1 +mongomock==4.3.0 more-itertools==10.5.0 moto==5.0.11 mpmath==1.3.0 msal-extensions==1.2.0 -msal==1.31.0 -msgraph-core==1.1.3 +msal==1.31.1 +msgraph-core==1.1.7 msrest==0.7.1 msrestazure==0.6.4.post1 multi_key_dict==2.0.3 multidict==6.1.0 -mypy-boto3-appflow==1.35.0 -mypy-boto3-rds==1.35.18 -mypy-boto3-redshift-data==1.35.10 -mypy-boto3-s3==1.35.16 +mypy-boto3-appflow==1.35.39 +mypy-boto3-rds==1.35.72 +mypy-boto3-redshift-data==1.35.51 +mypy-boto3-s3==1.35.76 mypy-extensions==1.0.0 mypy==1.9.0 -mysql-connector-python==9.0.0 -mysqlclient==2.2.4 -neo4j==5.24.0 -networkx==3.3 -nh3==0.2.18 +mysql-connector-python==9.1.0 +mysqlclient==2.2.6 +neo4j==5.27.0 +networkx==3.4.2 +nh3==0.2.19 nodeenv==1.9.1 numpy==1.26.4 oauthlib==3.2.2 -openai==1.45.1 +openai==1.57.2 openapi-schema-validator==0.6.2 openapi-spec-validator==0.7.1 -openlineage-integration-common==1.22.0 -openlineage-python==1.22.0 -openlineage_sql==1.22.0 +openlineage-integration-common==1.25.0 +openlineage-python==1.25.0 +openlineage_sql==1.25.0 openpyxl==3.1.5 -opensearch-py==2.7.1 +opensearch-py==2.8.0 opentelemetry-api==1.27.0 opentelemetry-exporter-otlp-proto-common==1.27.0 opentelemetry-exporter-otlp-proto-grpc==1.27.0 @@ -482,84 +489,85 @@ opentelemetry-proto==1.27.0 opentelemetry-sdk==1.27.0 opentelemetry-semantic-conventions==0.48b0 opsgenie-sdk==2.1.5 -oracledb==2.4.1 +oracledb==2.5.0 ordered-set==4.1.0 -oss2==2.19.0 -packaging==24.1 -pandas-gbq==0.23.1 -pandas-stubs==2.2.2.240909 +oss2==2.19.1 +packaging==24.2 +pandas-gbq==0.24.0 +pandas-stubs==2.2.3.241126 pandas==2.1.4 paramiko==3.5.0 parso==0.8.4 pathable==0.4.3 pathspec==0.12.1 pbr==6.1.0 -pdpyras==5.2.0 +pdpyras==5.3.0 pendulum==3.0.0 pexpect==4.9.0 -pgvector==0.3.3 +pgvector==0.3.6 pinecone-client==5.0.1 pinecone-plugin-inference==1.1.0 pinecone-plugin-interface==0.0.7 pinotdb==5.6.0 -pipdeptree==2.23.3 +pipdeptree==2.24.0 pipx==1.7.1 -pkginfo==1.10.0 -platformdirs==4.3.3 +pkginfo==1.12.0 +platformdirs==4.3.6 pluggy==1.5.0 ply==3.11 plyvel==1.5.1 portalocker==2.10.1 pprintpp==0.4.0 -pre-commit==3.8.0 +pre_commit==4.0.1 presto-python-client==0.8.4 prison==0.2.1 -prometheus_client==0.20.0 -prompt_toolkit==3.0.47 -proto-plus==1.24.0 -protobuf==4.25.4 -psutil==6.0.0 -psycopg2-binary==2.9.9 +prometheus_client==0.21.1 +prompt_toolkit==3.0.48 +propcache==0.2.1 +proto-plus==1.25.0 +protobuf==4.25.5 +psutil==6.1.0 +psycopg2-binary==2.9.10 ptyprocess==0.7.0 pure-sasl==0.6.2 pure_eval==0.2.3 py-partiql-parser==0.5.5 py4j==0.10.9.7 -pyOpenSSL==24.2.1 -pyarrow==17.0.0 +pyOpenSSL==24.3.0 +pyarrow==18.1.0 pyasn1==0.6.1 pyasn1_modules==0.4.0 pycountry==24.6.1 pycparser==2.22 -pycryptodome==3.20.0 -pydantic==2.9.1 -pydantic_core==2.23.3 -pydata-google-auth==1.8.2 +pycryptodome==3.21.0 +pydantic==2.10.3 +pydantic_core==2.27.1 +pydata-google-auth==1.9.0 pydruid==0.6.9 pyenchant==3.2.2 pyexasol==0.27.0 -pygraphviz==1.13 -pyiceberg==0.7.1 +pygraphviz==1.14 +pyiceberg==0.8.1 pykerberos==1.2.4 -pymongo==4.8.0 -pymssql==2.3.1 -pyodbc==5.1.0 -pyparsing==3.1.4 +pymongo==4.10.1 +pymssql==2.3.2 +pyodbc==5.2.0 +pyparsing==3.2.0 pypsrp==0.8.1 -pyspark==3.5.2 -pyspnego==0.11.1 +pyspark==3.5.3 +pyspnego==0.11.2 pytest-asyncio==0.24.0 -pytest-cov==5.0.0 +pytest-cov==6.0.0 pytest-custom-exit-code==0.3.0 pytest-icdiff==0.9 pytest-instafail==0.5.0 pytest-mock==3.14.0 -pytest-rerunfailures==14.0 +pytest-rerunfailures==15.0 pytest-timeouts==1.2.1 pytest-xdist==3.6.1 -pytest==8.3.3 -python-arango==8.1.1 -python-daemon==3.0.1 +pytest==8.3.4 +python-arango==8.1.4 +python-daemon==3.1.2 python-dateutil==2.9.0.post0 python-dotenv==1.0.1 python-http-client==3.3.7 @@ -567,17 +575,17 @@ python-jenkins==1.8.2 python-ldap==3.4.4 python-nvd3==0.16.0 python-slugify==8.0.4 -python-telegram-bot==21.5 +python-telegram-bot==21.9 python3-saml==1.16.0 pytz==2024.2 pywinrm==0.5.0 -qdrant-client==1.11.1 +qdrant-client==1.12.1 reactivex==4.0.4 readme_renderer==43.0 -redis==5.0.8 -redshift-connector==2.1.3 +redis==5.2.1 +redshift-connector==2.1.4 referencing==0.35.1 -regex==2024.9.11 +regex==2024.11.6 requests-file==2.1.0 requests-kerberos==0.15.0 requests-mock==1.12.1 @@ -589,32 +597,32 @@ responses==0.25.3 restructuredtext_lint==1.4.0 rfc3339-validator==0.1.4 rfc3986==2.0.0 -rich-argparse==1.5.2 -rich-click==1.8.3 -rich==13.8.1 -rpds-py==0.20.0 +rich-argparse==1.6.0 +rich-click==1.8.5 +rich==13.9.4 +rpds-py==0.22.3 rsa==4.9 ruff==0.5.5 -s3fs==2024.9.0 -s3transfer==0.10.2 +s3fs==2024.10.0 +s3transfer==0.10.4 scramp==1.4.5 semver==3.0.2 sendgrid==6.11.0 sentinels==1.0.0 -sentry-sdk==2.14.0 -setproctitle==1.3.3 -setuptools==75.1.0 +sentry-sdk==2.19.2 +setproctitle==1.3.4 +setuptools==75.6.0 shapely==2.0.6 shellingham==1.5.4 simple-salesforce==1.12.6 -six==1.16.0 -slack_sdk==3.32.0 -smbprotocol==1.14.0 +six==1.17.0 +slack_sdk==3.33.5 +smbprotocol==1.15.0 smmap==5.0.1 sniffio==1.3.1 snowballstemmer==2.2.0 -snowflake-connector-python==3.12.2 -snowflake-sqlalchemy==1.6.1 +snowflake-connector-python==3.12.4 +snowflake-sqlalchemy==1.7.1 sortedcontainers==2.4.0 soupsieve==2.6 sphinx-airflow-theme==0.0.12 @@ -635,90 +643,90 @@ sphinxcontrib-redoc==1.6.0 sphinxcontrib-serializinghtml==1.1.5 sphinxcontrib-spelling==8.0.0 spython==0.3.14 -sqlalchemy-bigquery==1.11.0 +sqlalchemy-bigquery==1.12.0 sqlalchemy-redshift==0.8.14 -sqlalchemy-spanner==1.7.0 +sqlalchemy-spanner==1.8.0 sqlalchemy_drill==1.1.5 -sqlparse==0.5.1 +sqlparse==0.5.3 sshtunnel==0.4.0 stack-data==0.6.3 starkbank-ecdsa==2.2.0 statsd==4.0.1 -std-uritemplate==1.0.6 +std-uritemplate==2.0.1 strictyaml==1.7.3 -sympy==1.13.2 -tableauserverclient==0.32 +sympy==1.13.3 +tableauserverclient==0.34 tabulate==0.9.0 -tenacity==8.5.0 -teradatasql==20.0.0.15 -teradatasqlalchemy==20.0.0.2 -termcolor==2.4.0 +tenacity==9.0.0 +teradatasql==20.0.0.20 +teradatasqlalchemy==20.0.0.3 +termcolor==2.5.0 text-unidecode==1.3 thrift-sasl==0.4.3 thrift==0.16.0 -time-machine==2.15.0 -tomli_w==1.0.0 +time-machine==2.16.0 +tomli_w==1.1.0 tomlkit==0.13.2 -tornado==6.4.1 +tornado==6.4.2 towncrier==24.8.0 -tqdm==4.66.5 +tqdm==4.67.1 traitlets==5.14.3 -trino==0.329.0 -trove-classifiers==2024.9.12 -twine==5.1.1 -typed_ast==1.5.5 -types-Deprecated==1.2.9.20240311 -types-Markdown==3.7.0.20240822 -types-PyMySQL==1.1.0.20240524 -types-PyYAML==6.0.12.20240808 +trino==0.331.0 +trove-classifiers==2024.10.21.16 +twine==6.0.1 +types-Deprecated==1.2.15.20241117 +types-Markdown==3.7.0.20241204 +types-PyMySQL==1.1.0.20241103 +types-PyYAML==6.0.12.20240917 types-aiofiles==24.1.0.20240626 types-certifi==2021.10.8.3 types-cffi==1.16.0.20240331 -types-croniter==3.0.3.20240731 -types-docutils==0.21.0.20240907 -types-paramiko==3.4.0.20240423 -types-protobuf==5.27.0.20240907 +types-croniter==5.0.1.20241205 +types-docutils==0.21.0.20241128 +types-paramiko==3.5.0.20240928 +types-protobuf==5.29.1.20241207 types-pyOpenSSL==24.1.0.20240722 -types-python-dateutil==2.9.0.20240906 +types-python-dateutil==2.9.0.20241206 types-python-slugify==8.0.2.20240310 -types-pytz==2024.2.0.20240913 -types-redis==4.6.0.20240903 -types-requests==2.32.0.20240914 -types-setuptools==74.1.0.20240907 -types-tabulate==0.9.0.20240106 +types-pytz==2024.2.0.20241003 +types-redis==4.6.0.20241004 +types-requests==2.32.0.20241016 +types-setuptools==75.6.0.20241126 +types-tabulate==0.9.0.20241207 types-termcolor==1.1.6.2 types-toml==0.10.8.20240310 +typing-inspect==0.9.0 typing_extensions==4.12.2 -tzdata==2024.1 +tzdata==2024.2 tzlocal==5.2 uc-micro-py==1.0.3 -unicodecsv==0.14.1 universal_pathlib==0.2.5 uritemplate==4.1.1 -urllib3==2.2.2 +urllib3==2.2.3 userpath==1.9.2 -uv==0.4.1 +uv==0.4.29 validators==0.33.0 vertica-python==1.4.0 vine==5.1.0 -virtualenv==20.26.4 +virtualenv==20.28.0 watchtower==3.3.1 wcwidth==0.2.13 weaviate-client==4.8.0 websocket-client==1.8.0 -wheel==0.44.0 -wirerope==0.4.7 -wrapt==1.16.0 +wheel==0.45.1 +wirerope==0.4.8 +wrapt==1.17.0 xmlsec==1.3.14 -xmltodict==0.13.0 +xmltodict==0.14.2 yamllint==1.35.1 yandex-query-client==0.1.4 -yandexcloud==0.316.0 -yarl==1.11.1 -ydb==3.17.1 -zeep==4.2.1 -zenpy==2.0.53 -zipp==3.20.2 +yandexcloud==0.328.0 +yarl==1.18.3 +ydb-dbapi==0.1.5 +ydb==3.18.10 +zeep==4.3.1 +zenpy==2.0.56 +zipp==3.21.0 zope.event==5.0 -zope.interface==7.0.3 +zope.interface==7.2 zstandard==0.23.0 diff --git a/airflow/constraints-2.9.2-python3.9.txt b/airflow/constraints-2.9.2-python3.9.txt deleted file mode 100644 index 691a8bf9d..000000000 --- a/airflow/constraints-2.9.2-python3.9.txt +++ /dev/null @@ -1,748 +0,0 @@ - -# -# This constraints file was automatically generated on 2024-06-06T07:19:45.961207 -# via "eager-upgrade" mechanism of PIP. For the "v2-9-test" branch of Airflow. -# This variant of constraints install uses the HEAD of the branch version for 'apache-airflow' but installs -# the providers from PIP-released packages at the moment of the constraint generation. -# -# Those constraints are actually those that regular users use to install released version of Airflow. -# We also use those constraints after "apache-airflow" is released and the constraints are tagged with -# "constraints-X.Y.Z" tag to build the production image for that version. -# -# This constraints file is meant to be used only in the "apache-airflow" installation command and not -# in all subsequent pip commands. By using a constraints.txt file, we ensure that solely the Airflow -# installation step is reproducible. Subsequent pip commands may install packages that would have -# been incompatible with the constraints used in Airflow reproducible installation step. Finally, pip -# commands that might change the installed version of apache-airflow should include "apache-airflow==X.Y.Z" -# in the list of install targets to prevent Airflow accidental upgrade or downgrade. -# -# Typical installation process of airflow for Python 3.8 is (with random selection of extras and custom -# dependencies added), usually consists of two steps: -# -# 1. Reproducible installation of airflow with selected providers (note constraints are used): -# -# pip install "apache-airflow[celery,cncf.kubernetes,google,amazon,snowflake]==X.Y.Z" \ -# --constraint \ -# "https://raw.githubusercontent.com/apache/airflow/constraints-X.Y.Z/constraints-3.9.txt" -# -# 2. Installing own dependencies that are potentially not matching the constraints (note constraints are not -# used, and apache-airflow==X.Y.Z is used to make sure there is no accidental airflow upgrade/downgrade. -# -# pip install "apache-airflow==X.Y.Z" "snowflake-connector-python[pandas]=N.M.O" -# -Authlib==1.3.1 -Babel==2.15.0 -ConfigUpdater==3.2 -Deprecated==1.2.14 -Events==0.5 -Flask-AppBuilder==4.4.1 -Flask-Babel==2.0.0 -Flask-Bcrypt==1.0.1 -Flask-Caching==2.3.0 -Flask-JWT-Extended==4.6.0 -Flask-Limiter==3.7.0 -Flask-Login==0.6.3 -Flask-SQLAlchemy==2.5.1 -Flask-Session==0.5.0 -Flask-WTF==1.2.1 -Flask==2.2.5 -GitPython==3.1.43 -JPype1==1.5.0 -JayDeBeApi==1.2.3 -Jinja2==3.1.4 -Js2Py==0.74 -Mako==1.3.5 -Markdown==3.6 -MarkupSafe==2.1.5 -PyAthena==3.8.3 -PyGithub==2.3.0 -PyHive==0.7.0 -PyJWT==2.8.0 -PyNaCl==1.5.0 -PyYAML==6.0.1 -Pygments==2.18.0 -SQLAlchemy-JSONField==1.0.2 -SQLAlchemy-Utils==0.41.2 -SQLAlchemy==1.4.52 -SecretStorage==3.3.3 -Sphinx==5.3.0 -WTForms==3.1.2 -Werkzeug==2.2.3 -adal==1.2.7 -adlfs==2024.4.1 -aiobotocore==2.13.0 -aiofiles==23.2.1 -aiohttp==3.9.5 -aioitertools==0.11.0 -aioresponses==0.7.6 -aiosignal==1.3.1 -alabaster==0.7.16 -alembic==1.13.1 -alibabacloud-adb20211201==1.3.5 -alibabacloud-tea==0.3.6 -alibabacloud_credentials==0.3.3 -alibabacloud_endpoint_util==0.0.3 -alibabacloud_gateway_spi==0.0.1 -alibabacloud_openapi_util==0.2.2 -alibabacloud_tea_openapi==0.3.9 -alibabacloud_tea_util==0.3.12 -alibabacloud_tea_xml==0.0.2 -aliyun-python-sdk-core==2.15.1 -aliyun-python-sdk-kms==2.16.3 -amqp==5.2.0 -analytics-python==1.2.9 -annotated-types==0.7.0 -ansicolors==1.1.8 -anyascii==0.3.2 -anyio==4.4.0 -apache-airflow-providers-airbyte==3.8.1 -apache-airflow-providers-alibaba==2.8.1 -apache-airflow-providers-amazon==8.24.0 -apache-airflow-providers-apache-beam==5.7.1 -apache-airflow-providers-apache-cassandra==3.5.1 -apache-airflow-providers-apache-drill==2.7.1 -apache-airflow-providers-apache-druid==3.10.1 -apache-airflow-providers-apache-flink==1.4.1 -apache-airflow-providers-apache-hdfs==4.4.1 -apache-airflow-providers-apache-hive==8.1.1 -apache-airflow-providers-apache-iceberg==1.0.0 -apache-airflow-providers-apache-impala==1.4.1 -apache-airflow-providers-apache-kafka==1.4.1 -apache-airflow-providers-apache-kylin==3.6.1 -apache-airflow-providers-apache-livy==3.8.1 -apache-airflow-providers-apache-pig==4.4.1 -apache-airflow-providers-apache-pinot==4.4.1 -apache-airflow-providers-apache-spark==4.8.1 -apache-airflow-providers-apprise==1.3.1 -apache-airflow-providers-arangodb==2.5.1 -apache-airflow-providers-asana==2.5.1 -apache-airflow-providers-atlassian-jira==2.6.1 -apache-airflow-providers-celery==3.7.2 -apache-airflow-providers-cloudant==3.5.1 -apache-airflow-providers-cncf-kubernetes==8.3.1 -apache-airflow-providers-cohere==1.2.1 -apache-airflow-providers-common-io==1.3.2 -apache-airflow-providers-common-sql==1.14.0 -apache-airflow-providers-databricks==6.5.0 -apache-airflow-providers-datadog==3.6.1 -apache-airflow-providers-dbt-cloud==3.8.1 -apache-airflow-providers-dingding==3.5.1 -apache-airflow-providers-discord==3.7.1 -apache-airflow-providers-docker==3.12.0 -apache-airflow-providers-elasticsearch==5.4.1 -apache-airflow-providers-exasol==4.5.1 -apache-airflow-providers-fab==1.1.1 -apache-airflow-providers-facebook==3.5.1 -apache-airflow-providers-ftp==3.9.1 -apache-airflow-providers-github==2.6.1 -apache-airflow-providers-google==10.19.0 -apache-airflow-providers-grpc==3.5.1 -apache-airflow-providers-hashicorp==3.7.1 -apache-airflow-providers-http==4.11.1 -apache-airflow-providers-imap==3.6.1 -apache-airflow-providers-influxdb==2.5.1 -apache-airflow-providers-jdbc==4.3.1 -apache-airflow-providers-jenkins==3.6.1 -apache-airflow-providers-microsoft-azure==10.1.1 -apache-airflow-providers-microsoft-mssql==3.7.1 -apache-airflow-providers-microsoft-psrp==2.7.1 -apache-airflow-providers-microsoft-winrm==3.5.1 -apache-airflow-providers-mongo==4.1.1 -apache-airflow-providers-mysql==5.6.1 -apache-airflow-providers-neo4j==3.6.1 -apache-airflow-providers-odbc==4.6.1 -apache-airflow-providers-openai==1.2.1 -apache-airflow-providers-openfaas==3.5.1 -apache-airflow-providers-openlineage==1.8.0 -apache-airflow-providers-opensearch==1.2.1 -apache-airflow-providers-opsgenie==5.6.1 -apache-airflow-providers-oracle==3.10.1 -apache-airflow-providers-pagerduty==3.7.1 -apache-airflow-providers-papermill==3.7.1 -apache-airflow-providers-pgvector==1.2.1 -apache-airflow-providers-pinecone==2.0.0 -apache-airflow-providers-postgres==5.11.1 -apache-airflow-providers-presto==5.5.1 -apache-airflow-providers-qdrant==1.1.1 -apache-airflow-providers-redis==3.7.1 -apache-airflow-providers-salesforce==5.7.1 -apache-airflow-providers-samba==4.7.1 -apache-airflow-providers-segment==3.5.1 -apache-airflow-providers-sendgrid==3.5.1 -apache-airflow-providers-sftp==4.10.1 -apache-airflow-providers-singularity==3.5.1 -apache-airflow-providers-slack==8.7.1 -apache-airflow-providers-smtp==1.7.1 -apache-airflow-providers-snowflake==5.5.1 -apache-airflow-providers-sqlite==3.8.1 -apache-airflow-providers-ssh==3.11.1 -apache-airflow-providers-tableau==4.5.1 -apache-airflow-providers-tabular==1.5.1 -apache-airflow-providers-telegram==4.5.1 -apache-airflow-providers-teradata==2.2.0 -apache-airflow-providers-trino==5.7.1 -apache-airflow-providers-vertica==3.8.1 -apache-airflow-providers-weaviate==1.4.1 -apache-airflow-providers-yandex==3.11.1 -apache-airflow-providers-zendesk==4.7.1 -apache-beam==2.56.0 -apispec==6.6.1 -apprise==1.8.0 -argcomplete==3.3.0 -asana==3.2.3 -asgiref==3.8.1 -asn1crypto==1.5.1 -astroid==2.15.8 -asttokens==2.4.1 -async-timeout==4.0.3 -asyncssh==2.14.2 -atlasclient==1.0.0 -atlassian-python-api==3.41.13 -attrs==23.2.0 -aws-sam-translator==1.89.0 -aws-xray-sdk==2.14.0 -azure-batch==14.2.0 -azure-common==1.1.28 -azure-core==1.30.1 -azure-cosmos==4.7.0 -azure-datalake-store==0.0.53 -azure-identity==1.16.0 -azure-keyvault-secrets==4.8.0 -azure-kusto-data==4.4.1 -azure-mgmt-containerinstance==10.1.0 -azure-mgmt-containerregistry==10.3.0 -azure-mgmt-core==1.4.0 -azure-mgmt-cosmosdb==9.5.0 -azure-mgmt-datafactory==7.1.0 -azure-mgmt-datalake-nspkg==3.0.1 -azure-mgmt-datalake-store==0.5.0 -azure-mgmt-nspkg==3.0.2 -azure-mgmt-resource==23.1.1 -azure-mgmt-storage==21.1.0 -azure-nspkg==3.0.2 -azure-servicebus==7.12.2 -azure-storage-blob==12.20.0 -azure-storage-file-datalake==12.15.0 -azure-storage-file-share==12.16.0 -azure-synapse-artifacts==0.19.0 -azure-synapse-spark==0.7.0 -backoff==2.2.1 -backports.tarfile==1.2.0 -bcrypt==4.1.3 -beautifulsoup4==4.12.3 -billiard==4.2.0 -bitarray==2.9.2 -black==24.4.2 -blinker==1.8.2 -boto3==1.34.106 -botocore==1.34.106 -cachelib==0.9.0 -cachetools==5.3.3 -cassandra-driver==3.29.1 -cattrs==23.2.3 -celery==5.4.0 -certifi==2024.6.2 -cffi==1.16.0 -cfgv==3.4.0 -cfn-lint==0.87.4 -cgroupspy==0.2.3 -chardet==5.2.0 -charset-normalizer==3.3.2 -checksumdir==1.2.0 -ciso8601==2.3.1 -click-didyoumean==0.3.1 -click-plugins==1.1.1 -click-repl==0.3.0 -click==8.1.7 -clickclick==20.10.2 -cloudant==2.15.0 -cloudpickle==2.2.1 -cohere==4.57 -colorama==0.4.6 -colorlog==4.8.0 -comm==0.2.2 -confluent-kafka==2.4.0 -connexion==2.14.2 -coverage==7.5.3 -crcmod==1.7 -cron-descriptor==1.4.3 -croniter==2.0.5 -cryptography==41.0.7 -curlify==2.2.1 -databricks-sql-connector==2.9.6 -datadog==0.49.1 -db-dtypes==1.2.0 -debugpy==1.8.1 -decorator==5.1.1 -defusedxml==0.7.1 -deltalake==0.17.4 -diagrams==0.23.4 -dill==0.3.1.1 -distlib==0.3.8 -distro==1.9.0 -dnspython==2.6.1 -docker==7.1.0 -docopt==0.6.2 -docstring_parser==0.16 -docutils==0.16 -duckdb==1.0.0 -elastic-transport==8.13.1 -elasticsearch==8.13.2 -email_validator==2.1.1 -entrypoints==0.4 -eralchemy2==1.4.1 -et-xmlfile==1.1.0 -eventlet==0.36.1 -exceptiongroup==1.2.1 -execnet==2.1.1 -executing==2.0.1 -facebook_business==19.0.3 -fastavro==1.9.4 -fasteners==0.19 -fastjsonschema==2.19.1 -filelock==3.14.0 -flower==2.0.1 -frozenlist==1.4.1 -fsspec==2023.12.2 -future==1.0.0 -gcloud-aio-auth==4.2.3 -gcloud-aio-bigquery==7.1.0 -gcloud-aio-storage==9.2.0 -gcsfs==2023.12.2.post1 -geomet==0.2.1.post1 -gevent==24.2.1 -gitdb==4.0.11 -google-ads==24.0.0 -google-analytics-admin==0.22.7 -google-api-core==2.19.0 -google-api-python-client==2.132.0 -google-auth-httplib2==0.2.0 -google-auth-oauthlib==1.2.0 -google-auth==2.29.0 -google-cloud-aiplatform==1.53.0 -google-cloud-appengine-logging==1.4.3 -google-cloud-audit-log==0.2.5 -google-cloud-automl==2.13.3 -google-cloud-batch==0.17.21 -google-cloud-bigquery-datatransfer==3.15.3 -google-cloud-bigquery==3.20.1 -google-cloud-bigtable==2.23.1 -google-cloud-build==3.24.0 -google-cloud-compute==1.19.0 -google-cloud-container==2.46.0 -google-cloud-core==2.4.1 -google-cloud-datacatalog==3.19.0 -google-cloud-dataflow-client==0.8.10 -google-cloud-dataform==0.5.9 -google-cloud-dataplex==2.0.0 -google-cloud-dataproc-metastore==1.15.3 -google-cloud-dataproc==5.9.3 -google-cloud-dlp==3.18.0 -google-cloud-kms==2.23.0 -google-cloud-language==2.13.3 -google-cloud-logging==3.10.0 -google-cloud-memcache==1.9.3 -google-cloud-monitoring==2.21.0 -google-cloud-orchestration-airflow==1.12.1 -google-cloud-os-login==2.14.3 -google-cloud-pubsub==2.21.2 -google-cloud-redis==2.15.3 -google-cloud-resource-manager==1.12.3 -google-cloud-run==0.10.5 -google-cloud-secret-manager==2.20.0 -google-cloud-spanner==3.47.0 -google-cloud-speech==2.26.0 -google-cloud-storage-transfer==1.11.3 -google-cloud-storage==2.16.0 -google-cloud-tasks==2.16.3 -google-cloud-texttospeech==2.16.3 -google-cloud-translate==3.15.3 -google-cloud-videointelligence==2.13.3 -google-cloud-vision==3.7.2 -google-cloud-workflows==1.14.3 -google-crc32c==1.5.0 -google-re2==1.1.20240601 -google-resumable-media==2.7.0 -googleapis-common-protos==1.63.1 -graphql-core==3.2.3 -graphviz==0.20.3 -greenlet==3.0.3 -grpc-google-iam-v1==0.13.0 -grpc-interceptor==0.15.4 -grpcio-gcp==0.2.2 -grpcio-status==1.62.2 -grpcio-tools==1.62.2 -grpcio==1.64.1 -gssapi==1.8.3 -gunicorn==22.0.0 -h11==0.14.0 -h2==4.1.0 -hatch==1.12.0 -hatchling==1.24.2 -hdfs==2.7.3 -hmsclient==0.1.1 -hpack==4.0.0 -httpcore==0.16.3 -httplib2==0.22.0 -httpx==0.23.3 -humanize==4.9.0 -hvac==2.2.0 -hyperframe==6.0.1 -hyperlink==21.0.0 -icdiff==2.0.7 -identify==2.5.36 -idna==3.7 -ijson==3.2.3 -imagesize==1.4.1 -importlib-metadata==6.11.0 -importlib_resources==6.4.0 -impyla==0.19.0 -incremental==22.10.0 -inflection==0.5.1 -influxdb-client==1.43.0 -iniconfig==2.0.0 -ipdb==0.13.13 -ipykernel==6.29.4 -ipython==8.18.1 -isodate==0.6.1 -itsdangerous==2.2.0 -jaraco.classes==3.4.0 -jaraco.context==5.3.0 -jaraco.functools==4.0.1 -jedi==0.19.1 -jeepney==0.8.0 -jmespath==0.10.0 -joserfc==0.11.1 -jschema-to-python==1.2.3 -json-merge-patch==0.2 -jsondiff==2.0.0 -jsonpatch==1.33 -jsonpath-ng==1.6.1 -jsonpickle==3.0.4 -jsonpointer==2.4 -jsonschema-path==0.3.2 -jsonschema-specifications==2023.12.1 -jsonschema==4.22.0 -junit-xml==1.9 -jupyter_client==8.6.2 -jupyter_core==5.7.2 -keyring==25.2.1 -kombu==5.3.7 -krb5==0.5.1 -kubernetes==29.0.0 -kubernetes_asyncio==29.0.0 -kylinpy==2.8.4 -lazy-object-proxy==1.10.0 -ldap3==2.9.1 -limits==3.12.0 -linkify-it-py==2.0.3 -lockfile==0.12.2 -loguru==0.7.2 -looker-sdk==24.8.0 -lxml==5.2.2 -lz4==4.3.3 -markdown-it-py==3.0.0 -marshmallow-oneofschema==3.1.1 -marshmallow-sqlalchemy==0.28.2 -marshmallow==3.21.3 -matplotlib-inline==0.1.7 -mdit-py-plugins==0.4.1 -mdurl==0.1.2 -methodtools==0.4.7 -microsoft-kiota-abstractions==1.3.3 -microsoft-kiota-authentication-azure==1.0.0 -microsoft-kiota-http==1.3.1 -mmhash3==3.0.1 -mongomock==4.1.2 -more-itertools==10.2.0 -moto==5.0.9 -mpmath==1.3.0 -msal-extensions==1.1.0 -msal==1.28.0 -msgraph-core==1.0.0 -msrest==0.7.1 -msrestazure==0.6.4.post1 -multi_key_dict==2.0.3 -multidict==6.0.5 -mypy-boto3-appflow==1.34.0 -mypy-boto3-rds==1.34.116 -mypy-boto3-redshift-data==1.34.0 -mypy-boto3-s3==1.34.120 -mypy-extensions==1.0.0 -mypy==1.9.0 -mysql-connector-python==8.4.0 -mysqlclient==2.2.4 -nbclient==0.10.0 -nbformat==5.10.4 -neo4j==5.20.0 -nest-asyncio==1.6.0 -networkx==3.2.1 -nh3==0.2.17 -nodeenv==1.9.1 -numpy==1.26.4 -oauthlib==3.2.2 -objsize==0.7.0 -openai==1.31.1 -openapi-schema-validator==0.6.2 -openapi-spec-validator==0.7.1 -openlineage-integration-common==1.16.0 -openlineage-python==1.16.0 -openlineage_sql==1.16.0 -openpyxl==3.1.3 -opensearch-py==2.6.0 -opentelemetry-api==1.25.0 -opentelemetry-exporter-otlp-proto-common==1.25.0 -opentelemetry-exporter-otlp-proto-grpc==1.25.0 -opentelemetry-exporter-otlp-proto-http==1.25.0 -opentelemetry-exporter-otlp==1.25.0 -opentelemetry-exporter-prometheus==0.46b0 -opentelemetry-proto==1.25.0 -opentelemetry-sdk==1.25.0 -opentelemetry-semantic-conventions==0.46b0 -opsgenie-sdk==2.1.5 -oracledb==2.2.1 -ordered-set==4.1.0 -orjson==3.10.3 -oss2==2.18.5 -packaging==24.0 -pandas-gbq==0.23.0 -pandas-stubs==2.2.2.240603 -pandas==2.1.4 -papermill==2.6.0 -paramiko==3.4.0 -parso==0.8.4 -pathable==0.4.3 -pathspec==0.12.1 -pbr==6.0.0 -pdpyras==5.2.0 -pendulum==3.0.0 -pexpect==4.9.0 -pgvector==0.2.5 -pinecone-client==4.1.1 -pinecone-plugin-interface==0.0.7 -pinotdb==5.2.0 -pipdeptree==2.22.0 -pipx==1.6.0 -pkginfo==1.11.0 -platformdirs==4.2.2 -pluggy==1.5.0 -ply==3.11 -plyvel==1.5.1 -portalocker==2.8.2 -pprintpp==0.4.0 -pre-commit==3.7.1 -presto-python-client==0.8.4 -prison==0.2.1 -prometheus_client==0.20.0 -prompt_toolkit==3.0.46 -proto-plus==1.23.0 -protobuf==4.25.3 -psutil==5.9.8 -psycopg2-binary==2.9.9 -ptyprocess==0.7.0 -pure-eval==0.2.2 -pure-sasl==0.6.2 -py-partiql-parser==0.5.5 -py4j==0.10.9.7 -pyOpenSSL==24.1.0 -pyarrow-hotfix==0.6 -pyarrow==14.0.2 -pyasn1-modules==0.3.0 -pyasn1==0.5.1 -pycountry==24.6.1 -pycparser==2.22 -pycryptodome==3.20.0 -pydantic==2.7.3 -pydantic_core==2.18.4 -pydata-google-auth==1.8.2 -pydot==1.4.2 -pydruid==0.6.9 -pyenchant==3.2.2 -pyexasol==0.25.2 -pygraphviz==1.11 -pyiceberg==0.6.1 -pyjsparser==2.7.1 -pykerberos==1.2.4 -pymongo==4.7.3 -pymssql==2.3.0 -pyodbc==5.1.0 -pyparsing==3.1.2 -pypsrp==0.8.1 -pyspark==3.5.1 -pyspnego==0.10.2 -pytest-asyncio==0.23.7 -pytest-cov==5.0.0 -pytest-custom-exit-code==0.3.0 -pytest-icdiff==0.9 -pytest-instafail==0.5.0 -pytest-mock==3.14.0 -pytest-rerunfailures==14.0 -pytest-timeouts==1.2.1 -pytest-xdist==3.6.1 -pytest==7.4.4 -python-arango==8.0.0 -python-daemon==3.0.1 -python-dateutil==2.9.0.post0 -python-dotenv==1.0.1 -python-http-client==3.3.7 -python-jenkins==1.8.2 -python-ldap==3.4.4 -python-nvd3==0.16.0 -python-slugify==8.0.4 -python-telegram-bot==20.2 -python3-saml==1.16.0 -pytz==2024.1 -pywinrm==0.4.3 -pyzmq==26.0.3 -qdrant-client==1.9.1 -reactivex==4.0.4 -readme_renderer==43.0 -redis==5.0.4 -redshift-connector==2.1.1 -referencing==0.31.1 -regex==2024.5.15 -requests-file==2.1.0 -requests-kerberos==0.15.0 -requests-mock==1.12.1 -requests-ntlm==1.2.0 -requests-oauthlib==1.3.1 -requests-toolbelt==1.0.0 -requests==2.31.0 -responses==0.25.0 -restructuredtext_lint==1.4.0 -rfc3339-validator==0.1.4 -rfc3986==1.5.0 -rich-argparse==1.5.0 -rich-click==1.8.2 -rich==13.7.1 -rpds-py==0.18.1 -rsa==4.9 -ruff==0.3.3 -s3fs==2023.12.2 -s3transfer==0.10.1 -sarif-om==1.0.4 -scramp==1.4.5 -scrapbook==0.5.0 -semver==3.0.2 -sendgrid==6.11.0 -sentinels==1.0.0 -sentry-sdk==2.4.0 -setproctitle==1.3.3 -shapely==2.0.4 -shellingham==1.5.4 -simple-salesforce==1.12.6 -six==1.16.0 -slack_sdk==3.27.2 -smbprotocol==1.13.0 -smmap==5.0.1 -sniffio==1.3.1 -snowballstemmer==2.2.0 -snowflake-connector-python==3.10.1 -snowflake-sqlalchemy==1.5.3 -sortedcontainers==2.4.0 -soupsieve==2.5 -sphinx-airflow-theme==0.0.12 -sphinx-argparse==0.4.0 -sphinx-autoapi==2.1.1 -sphinx-copybutton==0.5.2 -sphinx-jinja==2.0.2 -sphinx-rtd-theme==2.0.0 -sphinx_design==0.6.0 -sphinxcontrib-applehelp==1.0.8 -sphinxcontrib-devhelp==1.0.6 -sphinxcontrib-htmlhelp==2.0.5 -sphinxcontrib-httpdomain==1.8.1 -sphinxcontrib-jquery==4.1 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.7 -sphinxcontrib-redoc==1.6.0 -sphinxcontrib-serializinghtml==1.1.5 -sphinxcontrib-spelling==8.0.0 -spython==0.3.13 -sqlalchemy-bigquery==1.11.0 -sqlalchemy-redshift==0.8.14 -sqlalchemy-spanner==1.7.0 -sqlalchemy_drill==1.1.4 -sqlparse==0.5.0 -sshtunnel==0.4.0 -stack-data==0.6.3 -starkbank-ecdsa==2.2.0 -statsd==4.0.1 -std-uritemplate==0.0.57 -strictyaml==1.7.3 -sympy==1.12.1 -tableauserverclient==0.25 -tabulate==0.9.0 -tenacity==8.3.0 -teradatasql==20.0.0.12 -teradatasqlalchemy==20.0.0.1 -termcolor==2.4.0 -text-unidecode==1.3 -thrift-sasl==0.4.3 -thrift==0.16.0 -time-machine==2.14.1 -tomli==2.0.1 -tomli_w==1.0.0 -tomlkit==0.12.5 -tornado==6.4 -towncrier==23.11.0 -tqdm==4.66.4 -traitlets==5.14.3 -trino==0.328.0 -trove-classifiers==2024.5.22 -twine==5.1.0 -typed-ast==1.5.5 -types-Deprecated==1.2.9.20240311 -types-Markdown==3.6.0.20240316 -types-PyMySQL==1.1.0.20240524 -types-PyYAML==6.0.12.20240311 -types-aiofiles==23.2.0.20240403 -types-certifi==2021.10.8.3 -types-cffi==1.16.0.20240331 -types-croniter==2.0.0.20240423 -types-docutils==0.21.0.20240423 -types-paramiko==3.4.0.20240423 -types-protobuf==5.26.0.20240422 -types-pyOpenSSL==24.1.0.20240425 -types-python-dateutil==2.9.0.20240316 -types-python-slugify==8.0.2.20240310 -types-pytz==2024.1.0.20240417 -types-redis==4.6.0.20240425 -types-requests==2.31.0.6 -types-setuptools==70.0.0.20240524 -types-tabulate==0.9.0.20240106 -types-termcolor==1.1.6.2 -types-toml==0.10.8.20240310 -types-urllib3==1.26.25.14 -typing_extensions==4.12.1 -tzdata==2024.1 -tzlocal==5.2 -uc-micro-py==1.0.3 -unicodecsv==0.14.1 -universal_pathlib==0.2.2 -uritemplate==4.1.1 -urllib3==1.26.18 -userpath==1.9.2 -uv==0.2.2 -validators==0.28.3 -vertica-python==1.3.8 -vine==5.1.0 -virtualenv==20.26.2 -watchtower==3.2.0 -wcwidth==0.2.13 -weaviate-client==3.26.2 -websocket-client==1.8.0 -wirerope==0.4.7 -wrapt==1.16.0 -xmlsec==1.3.13 -xmltodict==0.13.0 -yamllint==1.35.1 -yandex-query-client==0.1.4 -yandexcloud==0.291.0 -yarl==1.9.4 -zeep==4.2.1 -zenpy==2.0.49 -zipp==3.19.2 -zope.event==5.0 -zope.interface==6.4.post2 -zstandard==0.22.0 diff --git a/airflow/download_constraints.sh b/airflow/download_constraints.sh index aa4f7f7b0..8ebc0824a 100755 --- a/airflow/download_constraints.sh +++ b/airflow/download_constraints.sh @@ -2,8 +2,8 @@ set -euo pipefail -AIRFLOW_VERSION=${1:?"Missing version number argument (arg 1)"} -PYTHON_VERSION=${2:-"3.9"} +AIRFLOW_VERSION=${1:?"Missing product version number argument (arg 1)"} +PYTHON_VERSION=${2:?"Missing python version number argument (arg 2)"} URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION}.txt" FILENAME="constraints-${AIRFLOW_VERSION}-python${PYTHON_VERSION}.txt" diff --git a/airflow/versions.py b/airflow/versions.py index 2d5364994..21b34efb8 100644 --- a/airflow/versions.py +++ b/airflow/versions.py @@ -1,12 +1,4 @@ versions = [ - { - "product": "2.9.2", - "python": "3.9", - "git_sync": "v4.4.0", - "statsd_exporter": "0.28.0", - "tini": "0.19.0", - "vector": "0.43.1", - }, { "product": "2.9.3", "python": "3.9", @@ -16,7 +8,7 @@ "vector": "0.43.1", }, { - "product": "2.10.2", + "product": "2.10.4", "python": "3.12", "git_sync": "v4.4.0", "statsd_exporter": "0.28.0", diff --git a/hadoop/stackable/patches/3.4.1/0001-YARN-11527-Update-node.js.patch b/hadoop/stackable/patches/3.4.1/0001-YARN-11527-Update-node.js.patch new file mode 100644 index 000000000..48840e282 --- /dev/null +++ b/hadoop/stackable/patches/3.4.1/0001-YARN-11527-Update-node.js.patch @@ -0,0 +1,22 @@ +From 8ebba29f08178fe26b3c0798d6d6b2c1a88578af Mon Sep 17 00:00:00 2001 +From: Siegfried Weber +Date: Thu, 21 Dec 2023 13:51:13 +0100 +Subject: YARN-11527: Update node.js + +--- + hadoop-project/pom.xml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml +index bcba56eced..3426a76c12 100644 +--- a/hadoop-project/pom.xml ++++ b/hadoop-project/pom.xml +@@ -217,7 +217,7 @@ + 1.0.2 + 5.4.0 + 9.37.2 +- v12.22.1 ++ v14.17.0 + v1.22.5 + 1.10.13 + 1.20 diff --git a/hadoop/stackable/patches/3.4.1/0002-Allow-overriding-datanode-registration-addresses.patch b/hadoop/stackable/patches/3.4.1/0002-Allow-overriding-datanode-registration-addresses.patch new file mode 100644 index 000000000..c18b2ee43 --- /dev/null +++ b/hadoop/stackable/patches/3.4.1/0002-Allow-overriding-datanode-registration-addresses.patch @@ -0,0 +1,259 @@ +From 52fecd5932904c1cd37d943d9a4f3ca3c8520dbc Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= +Date: Thu, 11 Jan 2024 14:01:02 +0100 +Subject: Allow overriding datanode registration addresses + +--- + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 9 +++ + .../blockmanagement/DatanodeManager.java | 43 +++++++----- + .../hadoop/hdfs/server/datanode/DNConf.java | 70 +++++++++++++++++++ + .../hadoop/hdfs/server/datanode/DataNode.java | 35 ++++++++-- + 4 files changed, 135 insertions(+), 22 deletions(-) + +diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +index f92a2ad565..25bcd438c7 100755 +--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ++++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +@@ -152,6 +152,13 @@ public class DFSConfigKeys extends CommonConfigurationKeys { + public static final boolean DFS_DATANODE_DROP_CACHE_BEHIND_READS_DEFAULT = false; + public static final String DFS_DATANODE_USE_DN_HOSTNAME = "dfs.datanode.use.datanode.hostname"; + public static final boolean DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT = false; ++ ++ public static final String DFS_DATANODE_REGISTERED_HOSTNAME = "dfs.datanode.registered.hostname"; ++ public static final String DFS_DATANODE_REGISTERED_DATA_PORT = "dfs.datanode.registered.port"; ++ public static final String DFS_DATANODE_REGISTERED_HTTP_PORT = "dfs.datanode.registered.http.port"; ++ public static final String DFS_DATANODE_REGISTERED_HTTPS_PORT = "dfs.datanode.registered.https.port"; ++ public static final String DFS_DATANODE_REGISTERED_IPC_PORT = "dfs.datanode.registered.ipc.port"; ++ + public static final String DFS_DATANODE_MAX_LOCKED_MEMORY_KEY = "dfs.datanode.max.locked.memory"; + public static final long DFS_DATANODE_MAX_LOCKED_MEMORY_DEFAULT = 0; + public static final String DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_KEY = "dfs.datanode.fsdatasetcache.max.threads.per.volume"; +@@ -491,6 +498,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { + public static final long DFS_DATANODE_PROCESS_COMMANDS_THRESHOLD_DEFAULT = + TimeUnit.SECONDS.toMillis(2); + ++ public static final String DFS_NAMENODE_DATANODE_REGISTRATION_UNSAFE_ALLOW_ADDRESS_OVERRIDE_KEY = "dfs.namenode.datanode.registration.unsafe.allow-address-override"; ++ public static final boolean DFS_NAMENODE_DATANODE_REGISTRATION_UNSAFE_ALLOW_ADDRESS_OVERRIDE_DEFAULT = false; + public static final String DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_KEY = "dfs.namenode.datanode.registration.ip-hostname-check"; + public static final boolean DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_DEFAULT = true; + +diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +index ebd2fa992e..c56f254478 100644 +--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java ++++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +@@ -181,6 +181,8 @@ public class DatanodeManager { + private boolean hasClusterEverBeenMultiRack = false; + + private final boolean checkIpHostnameInRegistration; ++ private final boolean allowRegistrationAddressOverride; ++ + /** + * Whether we should tell datanodes what to cache in replies to + * heartbeat messages. +@@ -314,6 +316,11 @@ public class DatanodeManager { + // Block invalidate limit also has some dependency on heartbeat interval. + // Check setBlockInvalidateLimit(). + setBlockInvalidateLimit(configuredBlockInvalidateLimit); ++ this.allowRegistrationAddressOverride = conf.getBoolean( ++ DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_UNSAFE_ALLOW_ADDRESS_OVERRIDE_KEY, ++ DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_UNSAFE_ALLOW_ADDRESS_OVERRIDE_DEFAULT); ++ LOG.info(DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_UNSAFE_ALLOW_ADDRESS_OVERRIDE_KEY ++ + "=" + allowRegistrationAddressOverride); + this.checkIpHostnameInRegistration = conf.getBoolean( + DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_KEY, + DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_DEFAULT); +@@ -1158,27 +1165,29 @@ public class DatanodeManager { + */ + public void registerDatanode(DatanodeRegistration nodeReg) + throws DisallowedDatanodeException, UnresolvedTopologyException { +- InetAddress dnAddress = Server.getRemoteIp(); +- if (dnAddress != null) { +- // Mostly called inside an RPC, update ip and peer hostname +- String hostname = dnAddress.getHostName(); +- String ip = dnAddress.getHostAddress(); +- if (checkIpHostnameInRegistration && !isNameResolved(dnAddress)) { +- // Reject registration of unresolved datanode to prevent performance +- // impact of repetitive DNS lookups later. +- final String message = "hostname cannot be resolved (ip=" +- + ip + ", hostname=" + hostname + ")"; +- LOG.warn("Unresolved datanode registration: " + message); +- throw new DisallowedDatanodeException(nodeReg, message); ++ if (!allowRegistrationAddressOverride) { ++ InetAddress dnAddress = Server.getRemoteIp(); ++ if (dnAddress != null) { ++ // Mostly called inside an RPC, update ip and peer hostname ++ String hostname = dnAddress.getHostName(); ++ String ip = dnAddress.getHostAddress(); ++ if (checkIpHostnameInRegistration && !isNameResolved(dnAddress)) { ++ // Reject registration of unresolved datanode to prevent performance ++ // impact of repetitive DNS lookups later. ++ final String message = "hostname cannot be resolved (ip=" ++ + ip + ", hostname=" + hostname + ")"; ++ LOG.warn("Unresolved datanode registration: " + message); ++ throw new DisallowedDatanodeException(nodeReg, message); ++ } ++ // update node registration with the ip and hostname from rpc request ++ nodeReg.setIpAddr(ip); ++ nodeReg.setPeerHostName(hostname); + } +- // update node registration with the ip and hostname from rpc request +- nodeReg.setIpAddr(ip); +- nodeReg.setPeerHostName(hostname); + } +- ++ + try { + nodeReg.setExportedKeys(blockManager.getBlockKeys()); +- ++ + // Checks if the node is not on the hosts list. If it is not, then + // it will be disallowed from registering. + if (!hostConfigManager.isIncluded(nodeReg)) { +diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java +index 21b92db307..5d3437239c 100644 +--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java ++++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java +@@ -101,6 +101,11 @@ public class DNConf { + final boolean syncOnClose; + final boolean encryptDataTransfer; + final boolean connectToDnViaHostname; ++ private final String registeredHostname; ++ private final int registeredDataPort; ++ private final int registeredHttpPort; ++ private final int registeredHttpsPort; ++ private final int registeredIpcPort; + final boolean overwriteDownstreamDerivedQOP; + private final boolean pmemCacheRecoveryEnabled; + +@@ -189,6 +194,11 @@ public class DNConf { + connectToDnViaHostname = getConf().getBoolean( + DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME, + DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT); ++ registeredHostname = getConf().get(DFSConfigKeys.DFS_DATANODE_REGISTERED_HOSTNAME); ++ registeredDataPort = getConf().getInt(DFSConfigKeys.DFS_DATANODE_REGISTERED_DATA_PORT, -1); ++ registeredHttpPort = getConf().getInt(DFSConfigKeys.DFS_DATANODE_REGISTERED_HTTP_PORT, -1); ++ registeredHttpsPort = getConf().getInt(DFSConfigKeys.DFS_DATANODE_REGISTERED_HTTPS_PORT, -1); ++ registeredIpcPort = getConf().getInt(DFSConfigKeys.DFS_DATANODE_REGISTERED_IPC_PORT, -1); + this.blockReportInterval = getConf().getLong( + DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, + DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT); +@@ -363,6 +373,66 @@ public class DNConf { + return connectToDnViaHostname; + } + ++ /** ++ * Returns a hostname to register with the cluster instead of the system ++ * hostname. ++ * This is an expert setting and can be used in multihoming scenarios to ++ * override the detected hostname. ++ * ++ * @return null if the system hostname should be used, otherwise a hostname ++ */ ++ public String getRegisteredHostname() { ++ return registeredHostname; ++ } ++ ++ /** ++ * Returns a port number to register with the cluster instead of the ++ * data port that the node is listening on. ++ * This is an expert setting and can be used in multihoming scenarios to ++ * override the detected port. ++ * ++ * @return -1 if the actual port should be used, otherwise a port number ++ */ ++ public int getRegisteredDataPort() { ++ return registeredDataPort; ++ } ++ ++ /** ++ * Returns a port number to register with the cluster instead of the ++ * HTTP port that the node is listening on. ++ * This is an expert setting and can be used in multihoming scenarios to ++ * override the detected port. ++ * ++ * @return -1 if the actual port should be used, otherwise a port number ++ */ ++ public int getRegisteredHttpPort() { ++ return registeredHttpPort; ++ } ++ ++ /** ++ * Returns a port number to register with the cluster instead of the ++ * HTTPS port that the node is listening on. ++ * This is an expert setting and can be used in multihoming scenarios to ++ * override the detected port. ++ * ++ * @return -1 if the actual port should be used, otherwise a port number ++ */ ++ public int getRegisteredHttpsPort() { ++ return registeredHttpsPort; ++ } ++ ++ /** ++ * Returns a port number to register with the cluster instead of the ++ * IPC port that the node is listening on. ++ * This is an expert setting and can be used in multihoming scenarios to ++ * override the detected port. ++ * ++ * @return -1 if the actual port should be used, otherwise a port number ++ */ ++ public int getRegisteredIpcPort() { ++ return registeredIpcPort; ++ } ++ + /** + * Returns socket timeout + * +diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +index 956f5bbe51..22ae127d98 100644 +--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java ++++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +@@ -135,6 +135,7 @@ import java.util.HashSet; + import java.util.Iterator; + import java.util.List; + import java.util.Map; ++import java.util.Optional; + import java.util.Map.Entry; + import java.util.Set; + import java.util.UUID; +@@ -2076,11 +2077,35 @@ public class DataNode extends ReconfigurableBase + NodeType.DATA_NODE); + } + +- DatanodeID dnId = new DatanodeID( +- streamingAddr.getAddress().getHostAddress(), hostName, +- storage.getDatanodeUuid(), getXferPort(), getInfoPort(), +- infoSecurePort, getIpcPort()); +- return new DatanodeRegistration(dnId, storageInfo, ++ String registeredHostname = Optional ++ .ofNullable(dnConf.getRegisteredHostname()) ++ .orElseGet(() -> streamingAddr.getAddress().getHostAddress()); ++ int registeredDataPort = dnConf.getRegisteredDataPort(); ++ if (registeredDataPort == -1) { ++ registeredDataPort = getXferPort(); ++ } ++ int registeredHttpPort = dnConf.getRegisteredHttpPort(); ++ if (registeredHttpPort == -1) { ++ registeredHttpPort = getInfoPort(); ++ } ++ int registeredHttpsPort = dnConf.getRegisteredHttpsPort(); ++ if (registeredHttpsPort == -1) { ++ registeredHttpsPort = getInfoSecurePort(); ++ } ++ int registeredIpcPort = dnConf.getRegisteredIpcPort(); ++ if (registeredIpcPort == -1) { ++ registeredIpcPort = getIpcPort(); ++ } ++ ++ DatanodeID dnId = new DatanodeID(registeredHostname, ++ registeredHostname, ++ storage.getDatanodeUuid(), ++ registeredDataPort, ++ registeredHttpPort, ++ registeredHttpsPort, ++ registeredIpcPort); ++ ++ return new DatanodeRegistration(dnId, storageInfo, + new ExportedBlockKeys(), VersionInfo.getVersion()); + } + diff --git a/hadoop/stackable/patches/3.4.1/0003-Async-profiler-also-grab-itimer-events.patch b/hadoop/stackable/patches/3.4.1/0003-Async-profiler-also-grab-itimer-events.patch new file mode 100644 index 000000000..132d1acd1 --- /dev/null +++ b/hadoop/stackable/patches/3.4.1/0003-Async-profiler-also-grab-itimer-events.patch @@ -0,0 +1,29 @@ +From 96f29246c6d71f756c0427cbcee05fa0e2596d91 Mon Sep 17 00:00:00 2001 +From: Siegfried Weber +Date: Tue, 6 Feb 2024 16:10:54 +0100 +Subject: Async-profiler: also grab itimer events + +--- + .../src/main/java/org/apache/hadoop/http/ProfileServlet.java | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java +index ce53274151..909892ff90 100644 +--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java ++++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java +@@ -76,6 +76,7 @@ import org.apache.hadoop.util.ProcessUtils; + * Following event types are supported (default is 'cpu') (NOTE: not all OS'es support all events) + * // Perf events: + * // cpu ++ * // itimer + * // page-faults + * // context-switches + * // cycles +@@ -118,6 +119,7 @@ public class ProfileServlet extends HttpServlet { + private enum Event { + + CPU("cpu"), ++ ITIMER("itimer"), + ALLOC("alloc"), + LOCK("lock"), + PAGE_FAULTS("page-faults"), diff --git a/hadoop/stackable/patches/3.4.1/0004-HDFS-17378-Fix-missing-operationType-for-some-operat.patch b/hadoop/stackable/patches/3.4.1/0004-HDFS-17378-Fix-missing-operationType-for-some-operat.patch new file mode 100644 index 000000000..0efa0988a --- /dev/null +++ b/hadoop/stackable/patches/3.4.1/0004-HDFS-17378-Fix-missing-operationType-for-some-operat.patch @@ -0,0 +1,200 @@ +From 7eaf9c59347a0f4756d1f56733d5f1d74b8c5c47 Mon Sep 17 00:00:00 2001 +From: Sebastian Bernauer +Date: Thu, 15 Feb 2024 15:33:43 +0100 +Subject: HDFS-17378: Fix missing operationType for some operations in + authorizer + +--- + .../hdfs/server/namenode/FSNamesystem.java | 41 +++++++++++-------- + 1 file changed, 24 insertions(+), 17 deletions(-) + +diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +index 277cc42222..75766bcd69 100644 +--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ++++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +@@ -2618,15 +2618,16 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, + * @throws IOException + */ + BlockStoragePolicy getStoragePolicy(String src) throws IOException { ++ final String operationName = "getStoragePolicy"; + checkOperation(OperationCategory.READ); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + readLock(); + try { + checkOperation(OperationCategory.READ); + return FSDirAttrOp.getStoragePolicy(dir, pc, blockManager, src); + } finally { +- readUnlock("getStoragePolicy"); ++ readUnlock(operationName); + } + } + +@@ -2646,15 +2647,16 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, + } + + long getPreferredBlockSize(String src) throws IOException { ++ final String operationName = "getPreferredBlockSize"; + checkOperation(OperationCategory.READ); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + readLock(); + try { + checkOperation(OperationCategory.READ); + return FSDirAttrOp.getPreferredBlockSize(dir, pc, src); + } finally { +- readUnlock("getPreferredBlockSize"); ++ readUnlock(operationName); + } + } + +@@ -2707,7 +2709,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, + boolean createParent, short replication, long blockSize, + CryptoProtocolVersion[] supportedVersions, String ecPolicyName, + String storagePolicy, boolean logRetryCache) throws IOException { +- + HdfsFileStatus status; + try { + status = startFileInt(src, permissions, holder, clientMachine, flag, +@@ -2727,6 +2728,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, + long blockSize, CryptoProtocolVersion[] supportedVersions, + String ecPolicyName, String storagePolicy, boolean logRetryCache) + throws IOException { ++ final String operationName = "create"; + if (NameNode.stateChangeLog.isDebugEnabled()) { + StringBuilder builder = new StringBuilder(); + builder.append("DIR* NameSystem.startFile: src=").append(src) +@@ -2764,7 +2766,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, + + checkOperation(OperationCategory.WRITE); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + writeLock(); + try { + checkOperation(OperationCategory.WRITE); +@@ -2827,7 +2829,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, + dir.writeUnlock(); + } + } finally { +- writeUnlock("create", getLockReportInfoSupplier(src, null, stat)); ++ writeUnlock(operationName, getLockReportInfoSupplier(src, null, stat)); + // There might be transactions logged while trying to recover the lease. + // They need to be sync'ed even when an exception was thrown. + if (!skipSync) { +@@ -2856,10 +2858,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, + */ + boolean recoverLease(String src, String holder, String clientMachine) + throws IOException { ++ final String operationName = "recoverLease"; + boolean skipSync = false; + checkOperation(OperationCategory.WRITE); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + writeLock(); + try { + checkOperation(OperationCategory.WRITE); +@@ -2880,7 +2883,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, + skipSync = true; + throw se; + } finally { +- writeUnlock("recoverLease"); ++ writeUnlock(operationName); + // There might be transactions logged while trying to recover the lease. + // They need to be sync'ed even when an exception was thrown. + if (!skipSync) { +@@ -3096,6 +3099,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, + final Set excludes, + final int numAdditionalNodes, final String clientName + ) throws IOException { ++ final String operationName = "getAdditionalDatanode"; + //check if the feature is enabled + dtpReplaceDatanodeOnFailure.checkEnabled(); + +@@ -3107,7 +3111,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, + final BlockType blockType; + checkOperation(OperationCategory.WRITE); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + readLock(); + try { + // Changing this operation category to WRITE instead of making getAdditionalDatanode as a +@@ -3133,7 +3137,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, + "src=%s, fileId=%d, blk=%s, clientName=%s, clientMachine=%s", + src, fileId, blk, clientName, clientMachine)); + } finally { +- readUnlock("getAdditionalDatanode"); ++ readUnlock(operationName); + } + + if (clientnode == null) { +@@ -3155,10 +3159,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, + */ + void abandonBlock(ExtendedBlock b, long fileId, String src, String holder) + throws IOException { ++ final String operationName = "abandonBlock"; + NameNode.stateChangeLog.debug("BLOCK* NameSystem.abandonBlock: {} of file {}", b, src); + checkOperation(OperationCategory.WRITE); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + writeLock(); + try { + checkOperation(OperationCategory.WRITE); +@@ -3167,7 +3172,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, + NameNode.stateChangeLog.debug( + "BLOCK* NameSystem.abandonBlock: {} is removed from pendingCreates", b); + } finally { +- writeUnlock("abandonBlock"); ++ writeUnlock(operationName); + } + getEditLog().logSync(); + } +@@ -3221,10 +3226,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, + boolean completeFile(final String src, String holder, + ExtendedBlock last, long fileId) + throws IOException { ++ final String operationName = "completeFile"; + boolean success = false; + checkOperation(OperationCategory.WRITE); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + writeLock(); + try { + checkOperation(OperationCategory.WRITE); +@@ -3232,7 +3238,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, + success = FSDirWriteFileOp.completeFile(this, pc, src, holder, last, + fileId); + } finally { +- writeUnlock("completeFile"); ++ writeUnlock(operationName); + } + getEditLog().logSync(); + if (success) { +@@ -3666,10 +3672,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, + */ + void fsync(String src, long fileId, String clientName, long lastBlockLength) + throws IOException { ++ final String operationName = "fsync"; + NameNode.stateChangeLog.info("BLOCK* fsync: " + src + " for " + clientName); + checkOperation(OperationCategory.WRITE); + final FSPermissionChecker pc = getPermissionChecker(); +- FSPermissionChecker.setOperationType(null); ++ FSPermissionChecker.setOperationType(operationName); + writeLock(); + try { + checkOperation(OperationCategory.WRITE); +@@ -3683,7 +3690,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, + } + FSDirWriteFileOp.persistBlocks(dir, src, pendingFile, false); + } finally { +- writeUnlock("fsync"); ++ writeUnlock(operationName); + } + getEditLog().logSync(); + } diff --git a/hadoop/stackable/patches/3.4.1/0005-Update-CycloneDX-plugin.patch b/hadoop/stackable/patches/3.4.1/0005-Update-CycloneDX-plugin.patch new file mode 100644 index 000000000..d0852adeb --- /dev/null +++ b/hadoop/stackable/patches/3.4.1/0005-Update-CycloneDX-plugin.patch @@ -0,0 +1,44 @@ +From e939d836a1f3594de5e49da83e7d206b6f42dd45 Mon Sep 17 00:00:00 2001 +From: Lukas Voetmand +Date: Fri, 6 Sep 2024 17:53:52 +0200 +Subject: Update CycloneDX plugin + +--- + pom.xml | 10 ++++++---- + 1 file changed, 6 insertions(+), 4 deletions(-) + +diff --git a/pom.xml b/pom.xml +index 16a3733b36..e4c544f6cd 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -118,7 +118,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x + 4.2.0 + 1.1.1 + 3.10.1 +- 2.7.10 ++ 2.8.0 + + bash + +@@ -770,6 +770,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x + org.cyclonedx + cyclonedx-maven-plugin + ${cyclonedx.version} ++ ++ application ++ 1.5 ++ false ++ + + + package +@@ -778,9 +783,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x + + + +- +- xml +- + + + diff --git a/hadoop/stackable/patches/3.4.1/patchable.toml b/hadoop/stackable/patches/3.4.1/patchable.toml new file mode 100644 index 000000000..6a697d142 --- /dev/null +++ b/hadoop/stackable/patches/3.4.1/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/apache/hadoop.git" +base = "4d7825309348956336b8f06a08322b78422849b1" diff --git a/hadoop/upload_new_hadoop_version.sh b/hadoop/upload_new_hadoop_version.sh index 3acf2deca..15f58a9f9 100755 --- a/hadoop/upload_new_hadoop_version.sh +++ b/hadoop/upload_new_hadoop_version.sh @@ -5,6 +5,11 @@ set -euo pipefail VERSION=${1:?"Missing version number argument (arg 1)"} NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"} +# We prefer fast downloads... +BASE_DOWNLOAD_URL="https://dlcdn.apache.org/hadoop/common" +# However, if the version is not available, use the slow archive instead: +# BASE_DOWNLOAD_URL="https://archive.apache.org/dist/hadoop/common" + read -r -s -p "Nexus Password: " NEXUS_PASSWORD echo "" @@ -34,14 +39,14 @@ cd "$WORK_DIR" || exit bin_file=hadoop-$VERSION.tar.gz src_file=hadoop-$VERSION-src.tar.gz -echo "Downloading Hadoop (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)" -curl --fail -LOs "https://archive.apache.org/dist/hadoop/common/hadoop-$VERSION/$bin_file" -curl --fail -LOs "https://archive.apache.org/dist/hadoop/common/hadoop-$VERSION/$bin_file.asc" -curl --fail -LOs "https://archive.apache.org/dist/hadoop/common/hadoop-$VERSION/$bin_file.sha512" +echo "Downloading Hadoop (if this fails, try switching the BASE_DOWNLOAD_URL to the archive)" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hadoop-$VERSION/$bin_file" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hadoop-$VERSION/$bin_file.asc" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hadoop-$VERSION/$bin_file.sha512" -curl --fail -LOs "https://archive.apache.org/dist/hadoop/common/hadoop-$VERSION/$src_file" -curl --fail -LOs "https://archive.apache.org/dist/hadoop/common/hadoop-$VERSION/$src_file.asc" -curl --fail -LOs "https://archive.apache.org/dist/hadoop/common/hadoop-$VERSION/$src_file.sha512" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hadoop-$VERSION/$src_file" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hadoop-$VERSION/$src_file.asc" +curl --fail -LO --progress-bar "${BASE_DOWNLOAD_URL}/hadoop-$VERSION/$src_file.sha512" # It is probably redundant to check both the checksum and the signature but it's cheap and why not @@ -53,7 +58,7 @@ if ! (sha512sum --tag "$bin_file" | diff - "$bin_file.sha512" && sha512sum --tag fi echo "Validating signatures" -echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://archive.apache.org/dist/hadoop/common/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://archive.apache.org/dist/hadoop/common/KEYS | gpg --import")' +echo "--> NOTE: Make sure you have downloaded and added the KEYS file (${BASE_DOWNLOAD_URL}/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using \"curl ${BASE_DOWNLOAD_URL}/KEYS | gpg --import\")" if ! (gpg --verify "$bin_file.asc" "$bin_file" 2> /dev/null && gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then echo "ERROR: One of the signatures could not be verified" diff --git a/hadoop/versions.py b/hadoop/versions.py index a9bd6bcc0..cb506b7b9 100644 --- a/hadoop/versions.py +++ b/hadoop/versions.py @@ -4,7 +4,7 @@ "java-base": "11", "java-devel": "11", "async_profiler": "2.9", - "jmx_exporter": "1.0.1-stackable", + "jmx_exporter": "1.1.0", "protobuf": "3.7.1", "hdfs_utils": "0.4.0", }, @@ -13,7 +13,7 @@ "java-base": "11", "java-devel": "11", "async_profiler": "2.9", - "jmx_exporter": "1.0.1-stackable", + "jmx_exporter": "1.1.0", "protobuf": "3.7.1", "hdfs_utils": "0.4.0", }, @@ -22,8 +22,17 @@ "java-base": "11", "java-devel": "11", "async_profiler": "2.9", - "jmx_exporter": "1.0.1-stackable", + "jmx_exporter": "1.1.0", "protobuf": "3.7.1", "hdfs_utils": "0.4.0", }, + { + "product": "3.4.1", + "java-base": "11", + "java-devel": "11", + "async_profiler": "2.9", + "jmx_exporter": "1.1.0", + "protobuf": "3.7.1", + "hdfs_utils": "0.4.1", + }, ] diff --git a/hbase/versions.py b/hbase/versions.py index d86649a36..57f6f7495 100644 --- a/hbase/versions.py +++ b/hbase/versions.py @@ -11,7 +11,7 @@ "phoenix": "5.2.0", "hbase_profile": "2.4", "hadoop": "3.3.6", - "jmx_exporter": "1.0.1-stackable", # update the stackable/jmx/config folder too + "jmx_exporter": "1.1.0", # update the stackable/jmx/config folder too "opa_authorizer": "", # only for HBase 2.6.1 "delete_caches": "true", }, diff --git a/hive/stackable/patches/4.0.0/patchable.toml b/hive/stackable/patches/4.0.0/patchable.toml deleted file mode 100644 index 7e70d2b90..000000000 --- a/hive/stackable/patches/4.0.0/patchable.toml +++ /dev/null @@ -1,2 +0,0 @@ -upstream = "https://github.com/apache/hive.git" -base = "183f8cb41d3dbed961ffd27999876468ff06690c" diff --git a/hive/stackable/patches/4.0.0/0001-Include-Postgres-driver.patch b/hive/stackable/patches/4.0.1/0001-Include-Postgres-driver.patch similarity index 87% rename from hive/stackable/patches/4.0.0/0001-Include-Postgres-driver.patch rename to hive/stackable/patches/4.0.1/0001-Include-Postgres-driver.patch index 50675ac24..c1c70fe7a 100644 --- a/hive/stackable/patches/4.0.0/0001-Include-Postgres-driver.patch +++ b/hive/stackable/patches/4.0.1/0001-Include-Postgres-driver.patch @@ -1,4 +1,4 @@ -From c5eb86648fe96b048723372024fa7278c9e108db Mon Sep 17 00:00:00 2001 +From 44d83fa02779ac3135f76ce711f11c108a432ea3 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 3 Sep 2024 11:13:24 +0200 Subject: Include Postgres driver @@ -9,7 +9,7 @@ Subject: Include Postgres driver 2 files changed, 2 deletions(-) diff --git a/standalone-metastore/metastore-server/pom.xml b/standalone-metastore/metastore-server/pom.xml -index a8f680928c..7102f1b5ca 100644 +index 3d14d6b2e3..fdc813c5b9 100644 --- a/standalone-metastore/metastore-server/pom.xml +++ b/standalone-metastore/metastore-server/pom.xml @@ -334,7 +334,6 @@ @@ -21,10 +21,10 @@ index a8f680928c..7102f1b5ca 100644 org.eclipse.jetty diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml -index 28ac5ceb65..e3cbd821bd 100644 +index 744b88c6db..c0bb928611 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml -@@ -397,7 +397,6 @@ +@@ -399,7 +399,6 @@ org.postgresql postgresql ${postgres.version} diff --git a/hive/stackable/patches/4.0.0/0002-Include-logging-dependencies.patch b/hive/stackable/patches/4.0.1/0002-Include-logging-dependencies.patch similarity index 87% rename from hive/stackable/patches/4.0.0/0002-Include-logging-dependencies.patch rename to hive/stackable/patches/4.0.1/0002-Include-logging-dependencies.patch index fc021b1e7..b63b54fb3 100644 --- a/hive/stackable/patches/4.0.0/0002-Include-logging-dependencies.patch +++ b/hive/stackable/patches/4.0.1/0002-Include-logging-dependencies.patch @@ -1,4 +1,4 @@ -From 69071d4d4525a8ceb27cbefa9a093d0678a1f3dd Mon Sep 17 00:00:00 2001 +From ed96173868c88142fded0291f901d84d6c63ed51 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Tue, 13 Aug 2024 13:38:12 +0200 Subject: Include logging dependencies @@ -9,10 +9,10 @@ This adds dependencies required for use of the XmlLayout for logging 1 file changed, 5 insertions(+) diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml -index e3cbd821bd..205fc31ec7 100644 +index c0bb928611..0f5de8ec3c 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml -@@ -493,6 +493,11 @@ +@@ -495,6 +495,11 @@ com.fasterxml.jackson.core jackson-databind diff --git a/hive/stackable/patches/4.0.0/0003-Add-CycloneDX-plugin.patch b/hive/stackable/patches/4.0.1/0003-Add-CycloneDX-plugin.patch similarity index 91% rename from hive/stackable/patches/4.0.0/0003-Add-CycloneDX-plugin.patch rename to hive/stackable/patches/4.0.1/0003-Add-CycloneDX-plugin.patch index 978de7fd3..1e4d28867 100644 --- a/hive/stackable/patches/4.0.0/0003-Add-CycloneDX-plugin.patch +++ b/hive/stackable/patches/4.0.1/0003-Add-CycloneDX-plugin.patch @@ -1,4 +1,4 @@ -From 23995b6c1ef70e4e119ce0493e63ff3a75ea1378 Mon Sep 17 00:00:00 2001 +From 45bac943ef6f6af5b357f1ed05182b011466fde3 Mon Sep 17 00:00:00 2001 From: Lukas Voetmand Date: Fri, 6 Sep 2024 17:53:52 +0200 Subject: Add CycloneDX plugin @@ -8,10 +8,10 @@ Subject: Add CycloneDX plugin 1 file changed, 18 insertions(+) diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml -index 205fc31ec7..2982a45ca0 100644 +index 0f5de8ec3c..edbac9ae11 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml -@@ -41,6 +41,7 @@ +@@ -42,6 +42,7 @@ false ${settings.localRepository} 3.1.0 @@ -19,7 +19,7 @@ index 205fc31ec7..2982a45ca0 100644 ${basedir}/${standalone.metastore.path.to.root}/checkstyle ${project.basedir}/src/test/resources -@@ -594,6 +595,23 @@ +@@ -595,6 +596,23 @@ diff --git a/hive/stackable/patches/4.0.0/0004-Fix-CVE-2024-36114.patch b/hive/stackable/patches/4.0.1/0004-Fix-CVE-2024-36114.patch similarity index 94% rename from hive/stackable/patches/4.0.0/0004-Fix-CVE-2024-36114.patch rename to hive/stackable/patches/4.0.1/0004-Fix-CVE-2024-36114.patch index d9d293e75..1eafef56c 100644 --- a/hive/stackable/patches/4.0.0/0004-Fix-CVE-2024-36114.patch +++ b/hive/stackable/patches/4.0.1/0004-Fix-CVE-2024-36114.patch @@ -1,4 +1,4 @@ -From 4a85ad5ec7b0dbfb9f2c4524531ae0198a352b3d Mon Sep 17 00:00:00 2001 +From 2b88bb3431c66a1173ba061f624a94073b447a08 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 12 Nov 2024 11:49:57 +0100 Subject: Fix CVE-2024-36114 @@ -26,10 +26,10 @@ known workarounds for this issue. 1 file changed, 6 insertions(+) diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml -index 2982a45ca0..cd34884e3b 100644 +index edbac9ae11..599ad33ed0 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml -@@ -121,6 +121,12 @@ +@@ -123,6 +123,12 @@ diff --git a/hive/stackable/patches/4.0.1/patchable.toml b/hive/stackable/patches/4.0.1/patchable.toml new file mode 100644 index 000000000..37091bbe1 --- /dev/null +++ b/hive/stackable/patches/4.0.1/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/apache/hive.git" +base = "3af4517eb8cfd9407ad34ed78a0b48b57dfaa264" diff --git a/hive/versions.py b/hive/versions.py index c35922942..10d4a47e8 100644 --- a/hive/versions.py +++ b/hive/versions.py @@ -1,8 +1,8 @@ versions = [ { - "product": "4.0.0", - "jmx_exporter": "1.0.1-stackable", - # Hive 4 must be built with Java 8 (according to GitHub README) but seems to run on Java 11 + "product": "3.1.3", + "jmx_exporter": "1.1.0", + # Hive 3 must be built with Java 8 but will run on Java 11 "java-base": "11", "java-devel": "8", "hadoop": "3.3.6", @@ -12,9 +12,9 @@ "azure_keyvault_core": "1.0.0", }, { - "product": "3.1.3", - "jmx_exporter": "1.0.1-stackable", - # Hive 3 must be built with Java 8 but will run on Java 11 + "product": "4.0.1", + "jmx_exporter": "1.1.0", + # Hive 4 must be built with Java 8 (according to GitHub README) but seems to run on Java 11 "java-base": "11", "java-devel": "8", "hadoop": "3.3.6", diff --git a/kafka/stackable/patches/3.8.1/0001-Change-Gradle-to-use-the-Nexus-Build-Repo.patch b/kafka/stackable/patches/3.8.0/001-cyclonedx-plugin.patch similarity index 78% rename from kafka/stackable/patches/3.8.1/0001-Change-Gradle-to-use-the-Nexus-Build-Repo.patch rename to kafka/stackable/patches/3.8.0/001-cyclonedx-plugin.patch index 7ce534d6c..f4587320f 100644 --- a/kafka/stackable/patches/3.8.1/0001-Change-Gradle-to-use-the-Nexus-Build-Repo.patch +++ b/kafka/stackable/patches/3.8.0/001-cyclonedx-plugin.patch @@ -1,14 +1,5 @@ -From b03f930266c43347645b4e978fa97cf644b270f8 Mon Sep 17 00:00:00 2001 -From: Lars Francke -Date: Thu, 12 Dec 2024 10:09:47 +0100 -Subject: Change Gradle to use the Nexus Build Repo - ---- - build.gradle | 44 +++++++++++++++++++++++++++++++++++++++++++- - 1 file changed, 43 insertions(+), 1 deletion(-) - diff --git a/build.gradle b/build.gradle -index 8b44b09087..fadb64fda5 100644 +index 92082fe..e3d6c72 100644 --- a/build.gradle +++ b/build.gradle @@ -48,6 +48,48 @@ plugins { @@ -60,12 +51,3 @@ index 8b44b09087..fadb64fda5 100644 } ext { -@@ -1011,7 +1053,7 @@ project(':core') { - testImplementation libs.junitJupiter - testImplementation libs.slf4jlog4j - testImplementation libs.caffeine -- -+ - generator project(':generator') - } - diff --git a/kafka/stackable/patches/3.8.1/0002-Change-Gradle-to-use-the-Nexus-Build-Repo.patch b/kafka/stackable/patches/3.8.0/002-use-stackable-repo.patch similarity index 77% rename from kafka/stackable/patches/3.8.1/0002-Change-Gradle-to-use-the-Nexus-Build-Repo.patch rename to kafka/stackable/patches/3.8.0/002-use-stackable-repo.patch index de47c472f..6c18efdaa 100644 --- a/kafka/stackable/patches/3.8.1/0002-Change-Gradle-to-use-the-Nexus-Build-Repo.patch +++ b/kafka/stackable/patches/3.8.0/002-use-stackable-repo.patch @@ -1,14 +1,14 @@ -From 698bad00ce09a01ee17664f7f3b04209d21603bc Mon Sep 17 00:00:00 2001 +From e5102449fe825cfbba20ce6ace1f51cd91550780 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 10:09:47 +0100 -Subject: Change Gradle to use the Nexus Build Repo +Subject: [PATCH] Change Gradle to use the Nexus Build Repo --- build.gradle | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle -index fadb64fda5..34786c0673 100644 +index 92082fe7cf..3b56a2ad98 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,9 @@ import java.nio.charset.StandardCharsets @@ -22,7 +22,7 @@ index fadb64fda5..34786c0673 100644 } apply from: "$rootDir/gradle/dependencies.gradle" -@@ -168,7 +170,9 @@ ext { +@@ -126,7 +128,9 @@ ext { allprojects { repositories { @@ -33,3 +33,6 @@ index fadb64fda5..34786c0673 100644 } dependencyUpdates { +-- +2.47.1 + diff --git a/kafka/stackable/patches/3.8.1/patchable.toml b/kafka/stackable/patches/3.8.1/patchable.toml deleted file mode 100644 index b2055980d..000000000 --- a/kafka/stackable/patches/3.8.1/patchable.toml +++ /dev/null @@ -1,2 +0,0 @@ -upstream = "https://github.com/apache/kafka.git" -base = "70d6ff42debf7e17478beb899fb5756bfbdbfbb5" diff --git a/kafka/versions.py b/kafka/versions.py index f2895dc58..5c0e64120 100644 --- a/kafka/versions.py +++ b/kafka/versions.py @@ -18,7 +18,7 @@ "jmx_exporter": "1.1.0", }, { - "product": "3.8.1", + "product": "3.8.0", "java-base": "21", "java-devel": "21", "scala": "2.13", diff --git a/omid/versions.py b/omid/versions.py index 4ba037180..1741353ab 100644 --- a/omid/versions.py +++ b/omid/versions.py @@ -3,6 +3,6 @@ "product": "1.1.2", "java-base": "11", "java-devel": "11", - "jmx_exporter": "1.0.1-stackable", + "jmx_exporter": "1.1.0", }, ] diff --git a/shell.nix b/shell.nix index 1782dff73..2ece43658 100644 --- a/shell.nix +++ b/shell.nix @@ -12,10 +12,13 @@ pkgs.mkShell { ]; buildInputs = [ - # Required for libraries to be discoverable - pkgs.pkg-config - # Required by patchable pkgs.openssl ]; + + nativeBuildInputs = [ + # Required for libraries to be discoverable + pkgs.pkg-config + ]; + } diff --git a/zookeeper/stackable/patches/3.9.3/0001-Add-CycloneDX-plugin.patch b/zookeeper/stackable/patches/3.9.3/0001-Add-CycloneDX-plugin.patch new file mode 100644 index 000000000..901191646 --- /dev/null +++ b/zookeeper/stackable/patches/3.9.3/0001-Add-CycloneDX-plugin.patch @@ -0,0 +1,34 @@ +From f2dbb32161000b95032fbc6caee276f2c92552d8 Mon Sep 17 00:00:00 2001 +From: Lukas Voetmand +Date: Fri, 6 Sep 2024 17:53:52 +0200 +Subject: Add CycloneDX plugin + +--- + pom.xml | 7 ++++++- + 1 file changed, 6 insertions(+), 1 deletion(-) + +diff --git a/pom.xml b/pom.xml +index 6ef4011f..07ae7538 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -925,7 +925,7 @@ + + org.cyclonedx + cyclonedx-maven-plugin +- 2.7.9 ++ 2.8.0 + + + +@@ -1200,6 +1200,11 @@ + + org.cyclonedx + cyclonedx-maven-plugin ++ ++ application ++ 1.5 ++ false ++ + + + diff --git a/zookeeper/stackable/patches/3.9.3/patchable.toml b/zookeeper/stackable/patches/3.9.3/patchable.toml new file mode 100644 index 000000000..dd391da25 --- /dev/null +++ b/zookeeper/stackable/patches/3.9.3/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/apache/zookeeper.git" +base = "c26634f34490bb0ea7a09cc51e05ede3b4e320ee" diff --git a/zookeeper/versions.py b/zookeeper/versions.py index 81af6bbf8..f7b7387fd 100644 --- a/zookeeper/versions.py +++ b/zookeeper/versions.py @@ -7,6 +7,16 @@ # zookeeper: Execution spotbugs of goal com.github.spotbugs:spotbugs-maven-plugin:4.0.0:spotbugs failed: Java # returned: 1 -> [Help 1] "java-devel": "11", - "jmx_exporter": "1.0.1-stackable", + "jmx_exporter": "1.1.0", + }, + { + "product": "3.9.3", + "java-base": "17", + # NOTE (@NickLarsenNZ): Builds fail on Java 17, with the output: + # [ERROR] Failed to execute goal com.github.spotbugs:spotbugs-maven-plugin:4.0.0:spotbugs (spotbugs) on project + # zookeeper: Execution spotbugs of goal com.github.spotbugs:spotbugs-maven-plugin:4.0.0:spotbugs failed: Java + # returned: 1 -> [Help 1] + "java-devel": "11", + "jmx_exporter": "1.1.0", }, ]