fix

Author: yuzelin

.github/workflows/check-java-bridge-licensing.yml

@@ -40,14 +40,14 @@ jobs:
       - name: Build
         run: |
           set -o pipefail
-          cd pypaimon/py4j/paimon-python-java-bridge
+          cd paimon-python-java-bridge
           mvn clean deploy ${{ env.MVN_COMMON_OPTIONS }} -DskipTests \
             -DaltDeploymentRepository=validation_repository::default::file:${{ env.MVN_VALIDATION_DIR }} \
             | tee ${{ env.MVN_BUILD_OUTPUT_FILE }}
 
       - name: Check licensing
         run: |
-          cd pypaimon/py4j/paimon-python-java-bridge
+          cd paimon-python-java-bridge
           mvn ${{ env.MVN_COMMON_OPTIONS }} exec:java@check-licensing -N \
             -Dexec.args="${{ env.MVN_BUILD_OUTPUT_FILE }} $(pwd) ${{ env.MVN_VALIDATION_DIR }}" \
             -Dlog4j.configurationFile=file://$(pwd)/tools/ci/log4j.properties

dev/lint-python.sh

@@ -577,6 +577,10 @@ function tox_check() {
     # Ensure the permission of the scripts set correctly
     chmod +x $PAIMON_PYTHON_DIR/dev/*
 
+    # dummy jar needed by setup.py
+    mkdir -p $PAIMON_PYTHON_DIR/deps/jars
+    touch $PAIMON_PYTHON_DIR/deps/jars/dummy.jar
+
     if [[ -n "$GITHUB_ACTION" ]]; then
         # Run tests in all versions triggered by a Git push (tests aren't so many currently)
         $TOX_PATH -vv -c $PAIMON_PYTHON_DIR/tox.ini --recreate 2>&1 | tee -a $LOG_FILE
@@ -593,6 +597,9 @@ function tox_check() {
         $TOX_PATH -vv -c $PAIMON_PYTHON_DIR/tox.ini -e ${ENV_LIST[$index]} --recreate 2>&1 | tee -a $LOG_FILE
     fi
 
+    # delete dummy jar
+    rm -rf $PAIMON_PYTHON_DIR/deps
+
     TOX_RESULT=$((grep -c "congratulations :)" "$LOG_FILE") 2>&1)
     if [ $TOX_RESULT -eq '0' ]; then
         print_function "STAGE" "tox checks... [FAILED]"

dev/test_deps/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar

@@ -18,65 +18,10 @@
 
 import os
 import shutil
-import subprocess
 import tempfile
 import unittest
-import urllib.request
 
 from pypaimon.py4j import constants, Catalog
-from xml.etree import ElementTree
-
-
-def _setup_hadoop_bundle_jar(hadoop_dir):
-    url = 'https://repo.maven.apache.org/maven2/org/apache/flink/' \
-          'flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar'
-
-    response = urllib.request.urlopen(url)
-    if not os.path.exists(hadoop_dir):
-        os.mkdir(hadoop_dir)
-
-    jar_path = os.path.join(hadoop_dir, "bundled-hadoop.jar")
-    with open(jar_path, 'wb') as file:
-        file.write(response.read())
-
-    os.environ[constants.PYPAIMON_HADOOP_CLASSPATH] = jar_path
-
-
-def _setup_bridge_jar(bridge_dir):
-    java_bridge_module = _find_java_bridge_module()
-    subprocess.run(
-        ["mvn", "clean", "package"],
-        cwd=java_bridge_module,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE
-    )
-
-    if not os.path.exists(bridge_dir):
-        os.mkdir(bridge_dir)
-
-    jar_path = os.path.join(bridge_dir, "paimon-python-java-bridge.jar")
-    shutil.copy(
-        os.path.join(java_bridge_module, 'target/{}-{}.jar'
-                     .format('paimon-python-java-bridge', _extract_bridge_version())),
-        jar_path
-    )
-
-    os.environ[constants.PYPAIMON_JAVA_CLASSPATH] = jar_path
-
-
-def _extract_bridge_version():
-    pom_path = os.path.join(_find_java_bridge_module(), 'pom.xml')
-    return ElementTree.parse(pom_path).getroot().find(
-        'POM:version',
-        namespaces={
-            'POM': 'http://maven.apache.org/POM/4.0.0'
-        }).text
-
-
-def _find_java_bridge_module():
-    this_dir = os.path.abspath(os.path.dirname(__file__))
-    project_dir = os.path.dirname(os.path.dirname(os.path.dirname(this_dir)))
-    return os.path.join(project_dir, "paimon-python-java-bridge")
 
 
 class PypaimonTestBase(unittest.TestCase):
@@ -87,11 +32,12 @@ class PypaimonTestBase(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         os.environ[constants.PYPAIMON4J_TEST_MODE] = 'true'
-        cls.tempdir = tempfile.mkdtemp()
 
-        _setup_hadoop_bundle_jar(cls.tempdir)
-        _setup_bridge_jar(cls.tempdir)
+        this_dir = os.path.abspath(os.path.dirname(__file__))
+        project_dir = os.path.dirname(os.path.dirname(os.path.dirname(this_dir)))
+        os.environ[constants.PYPAIMON_HADOOP_CLASSPATH] = os.path.join(project_dir, "dev/test_deps/*")
 
+        cls.tempdir = tempfile.mkdtemp()
         cls.warehouse = os.path.join(cls.tempdir, 'warehouse')
         cls.catalog = Catalog.create({'warehouse': cls.warehouse})
         cls.catalog.create_database('default', False)

dev/test_deps/paimon-python-java-bridge-0.9-SNAPSHOT.jar

@@ -37,7 +37,6 @@ def setUpClass(cls):
             ('f1', pa.string())
         ])
 
-
     def testReadEmptyAppendTable(self):
         schema = Schema(self.simple_pa_schema)
         self.catalog.create_table('default.empty_append_table', schema, False)