Remove flink-shaded-hadoop-2-uber in java-bridge (#14)

Author: yuzelin

paimon_python_java/gateway_server.py

@@ -79,12 +79,30 @@ def preexec_func():
 
 
 def _get_classpath(env):
-    user_defined = env.get(constants.PYPAIMON_JAVA_CLASSPATH)
+    classpath = []
 
     module = importlib.import_module(_JAVA_IMPL_MODULE)
     builtin_java_bridge = os.path.join(*module.__path__, _JAVA_DEPS, _JAVA_BRIDGE + '.jar')
+    classpath.append(builtin_java_bridge)
 
-    if user_defined is None:
-        return builtin_java_bridge
+    # user defined
+    if constants.PYPAIMON_JAVA_CLASSPATH in env:
+        classpath.append(env[constants.PYPAIMON_JAVA_CLASSPATH])
+
+    # hadoop
+    hadoop_classpath = _get_hadoop_classpath(env)
+    if hadoop_classpath is not None:
+        classpath.append(hadoop_classpath)
+
+    return os.pathsep.join(classpath)
+
+
+def _get_hadoop_classpath(env):
+    if constants.PYPAIMON_HADOOP_CLASSPATH in env:
+        return env[constants.PYPAIMON_HADOOP_CLASSPATH]
+
+    if 'HADOOP_CLASSPATH' in env:
+        return None
     else:
-        return os.pathsep.join([builtin_java_bridge, user_defined])
+        raise EnvironmentError(f"You haven't set '{constants.PYPAIMON_HADOOP_CLASSPATH}', \
+ and 'HADOOP_CLASSPATH' is also not set. Ensure one of them is set.")

paimon_python_java/paimon-python-java-bridge/pom.xml

@@ -30,7 +30,6 @@
 
     <properties>
         <paimon.version>0.9.0</paimon.version>
-        <flink.shaded.hadoop.version>2.8.3-10.0</flink.shaded.hadoop.version>
         <py4j.version>0.10.9.7</py4j.version>
         <slf4j.version>1.7.32</slf4j.version>
         <log4j.version>2.17.1</log4j.version>
@@ -68,12 +67,6 @@
             <version>${log4j.version}</version>
         </dependency>
 
-        <dependency>
-            <groupId>org.apache.flink</groupId>
-            <artifactId>flink-shaded-hadoop-2-uber</artifactId>
-            <version>${flink.shaded.hadoop.version}</version>
-        </dependency>
-
         <dependency>
             <groupId>org.apache.arrow</groupId>
             <artifactId>arrow-vector</artifactId>
@@ -178,7 +171,6 @@
                                     <include>com.google.flatbuffers:flatbuffers-java</include>
                                     <include>org.slf4j:slf4j-api</include>
                                     <include>org.apache.logging.log4j:log4j-1.2-api</include>
-                                    <include>org.apache.flink:flink-shaded-hadoop-2-uber</include>
                                     <include>net.sf.py4j:py4j</include>
                                 </includes>
                             </artifactSet>

paimon_python_java/tests/test_data_types.py

@@ -24,9 +24,9 @@
 import pyarrow as pa
 import unittest
 
-
 from paimon_python_api import Schema
 from paimon_python_java import Catalog
+from paimon_python_java.tests import utils
 from paimon_python_java.util import java_utils
 from setup_utils import java_setuputils
 
@@ -36,6 +36,8 @@ class DataTypesTest(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         java_setuputils.setup_java_bridge()
+        cls.hadoop_path = tempfile.mkdtemp()
+        utils.setup_hadoop_bundle_jar(cls.hadoop_path)
         cls.warehouse = tempfile.mkdtemp()
         cls.simple_pa_schema = pa.schema([
             ('f0', pa.int32()),
@@ -47,6 +49,8 @@ def setUpClass(cls):
     @classmethod
     def tearDownClass(cls):
         java_setuputils.clean()
+        if os.path.exists(cls.hadoop_path):
+            shutil.rmtree(cls.hadoop_path)
         if os.path.exists(cls.warehouse):
             shutil.rmtree(cls.warehouse)
 

paimon_python_java/tests/test_write_and_read.py

@@ -22,20 +22,23 @@
 import unittest
 import pandas as pd
 import pyarrow as pa
-import setup_utils.java_setuputils as setuputils
 
 from paimon_python_api import Schema
 from paimon_python_java import Catalog
 from paimon_python_java.java_gateway import get_gateway
+from paimon_python_java.tests import utils
 from paimon_python_java.util import java_utils
 from py4j.protocol import Py4JJavaError
+from setup_utils import java_setuputils
 
 
 class TableWriteReadTest(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        setuputils.setup_java_bridge()
+        java_setuputils.setup_java_bridge()
+        cls.hadoop_path = tempfile.mkdtemp()
+        utils.setup_hadoop_bundle_jar(cls.hadoop_path)
         cls.warehouse = tempfile.mkdtemp()
         cls.simple_pa_schema = pa.schema([
             ('f0', pa.int32()),
@@ -46,7 +49,9 @@ def setUpClass(cls):
 
     @classmethod
     def tearDownClass(cls):
-        setuputils.clean()
+        java_setuputils.clean()
+        if os.path.exists(cls.hadoop_path):
+            shutil.rmtree(cls.hadoop_path)
         if os.path.exists(cls.warehouse):
             shutil.rmtree(cls.warehouse)
 

paimon_python_java/tests/utils.py

@@ -0,0 +1,37 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import os
+import urllib.request
+
+from paimon_python_java.util import constants
+
+
+def setup_hadoop_bundle_jar(hadoop_dir):
+    url = 'https://repo.maven.apache.org/maven2/org/apache/flink/' \
+          'flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar'
+
+    response = urllib.request.urlopen(url)
+    if not os.path.exists(hadoop_dir):
+        os.mkdir(hadoop_dir)
+
+    jar_path = os.path.join(hadoop_dir, "bundled-hadoop.jar")
+    with open(jar_path, 'wb') as file:
+        file.write(response.read())
+
+    os.environ[constants.PYPAIMON_HADOOP_CLASSPATH] = jar_path

paimon_python_java/util/constants.py

@@ -20,6 +20,7 @@
 PYPAIMON_CONN_INFO_PATH = '_PYPAIMON_CONN_INFO_PATH'
 PYPAIMON_JVM_ARGS = '_PYPAIMON_JVM_ARGS'
 PYPAIMON_JAVA_CLASSPATH = '_PYPAIMON_JAVA_CLASSPATH'
+PYPAIMON_HADOOP_CLASSPATH = '_PYPAIMON_HADOOP_CLASSPATH'
 PYPAIMON_MAIN_CLASS = 'org.apache.paimon.python.PythonGatewayServer'
 PYPAIMON_MAIN_ARGS = '_PYPAIMON_MAIN_ARGS'