Skip to content

Commit 9c6a60b

Browse files
chenghuichenchenghuichen
committed
#40 Provide Optional Built-in Hadoop Dependencies
1 parent 75d00d7 commit 9c6a60b

File tree

6 files changed

+126
-19
lines changed

6 files changed

+126
-19
lines changed

‎MANIFEST.in‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
global-exclude *.py[cod] __pycache__ .DS_Store
2020
recursive-include deps/jars *.jar
21+
recursive-include deps/hadoop *.jar
2122
include README.md
2223
include LICENSE
2324
include NOTICE

‎dev/build-source-distribution-package.sh‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,20 +19,34 @@ CURR_DIR=`pwd`
1919
BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
2020
PROJECT_ROOT="${BASE_DIR}/../"
2121

22-
# prepare bridge jar
23-
24-
DEPS_DIR=${PROJECT_ROOT}/deps/jars
22+
DEPS_DIR=${PROJECT_ROOT}/deps
2523
rm -rf ${DEPS_DIR}
26-
mkdir -p ${DEPS_DIR}
27-
touch ${DEPS_DIR}/__init__.py
24+
25+
# prepare bridge jar
26+
BRIDGE_DEPS_DIR=${DEPS_DIR}/jars
27+
mkdir -p ${BRIDGE_DEPS_DIR}
28+
touch ${BRIDGE_DEPS_DIR}/__init__.py
2829

2930
cd ${PROJECT_ROOT}/paimon-python-java-bridge
3031

3132
# get bridge jar version
32-
JAR_VERSION=$(sed -n 's/.*<version>\(.*\)<\/version>.*/\1/p' pom.xml | head -n 1)
33+
BRIDGE_JAR_VERSION=$(sed -n 's/.*<version>\(.*\)<\/version>.*/\1/p' pom.xml | head -n 1)
34+
35+
mvn clean install -DskipTests
36+
cp "target/paimon-python-java-bridge-${BRIDGE_JAR_VERSION}.jar" ${BRIDGE_DEPS_DIR}
37+
38+
# prepare hadoop-deps jar
39+
HADOOP_DEPS_DIR=${DEPS_DIR}/hadoop
40+
mkdir -p ${HADOOP_DEPS_DIR}
41+
touch ${HADOOP_DEPS_DIR}/__init__.py
42+
43+
cd ${PROJECT_ROOT}/hadoop-deps
44+
45+
# get hadoop-deps jar version
46+
HADOOP_JAR_VERSION=$(sed -n 's/.*<version>\(.*\)<\/version>.*/\1/p' pom.xml | head -n 1)
3347

3448
mvn clean install -DskipTests
35-
cp "target/paimon-python-java-bridge-${JAR_VERSION}.jar" ${DEPS_DIR}
49+
cp "target/hadoop-deps-${HADOOP_JAR_VERSION}.jar" ${HADOOP_DEPS_DIR}
3650

3751
cd ${CURR_DIR}
3852

‎hadoop-deps/pom.xml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one
4+
~ or more contributor license agreements. See the NOTICE file
5+
~ distributed with this work for additional information
6+
~ regarding copyright ownership. The ASF licenses this file
7+
~ to you under the Apache License, Version 2.0 (the
8+
~ "License"); you may not use this file except in compliance
9+
~ with the License. You may obtain a copy of the License at
10+
~
11+
~ http://www.apache.org/licenses/LICENSE-2.0
12+
~
13+
~ Unless required by applicable law or agreed to in writing, software
14+
~ distributed under the License is distributed on an "AS IS" BASIS,
15+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
~ See the License for the specific language governing permissions and
17+
~ limitations under the License.
18+
-->
19+
<project xmlns="http://maven.apache.org/POM/4.0.0"
20+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
21+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
22+
<modelVersion>4.0.0</modelVersion>
23+
24+
<groupId>org.apache.pypaimon</groupId>
25+
<artifactId>hadoop-deps</artifactId>
26+
<version>3.3.4</version>
27+
28+
<properties>
29+
<hadoop.version>3.3.4</hadoop.version>
30+
</properties>
31+
32+
<dependencies>
33+
<dependency>
34+
<groupId>org.apache.hadoop</groupId>
35+
<artifactId>hadoop-common</artifactId>
36+
<version>${hadoop.version}</version>
37+
</dependency>
38+
<dependency>
39+
<groupId>org.apache.hadoop</groupId>
40+
<artifactId>hadoop-hdfs-client</artifactId>
41+
<version>${hadoop.version}</version>
42+
</dependency>
43+
</dependencies>
44+
45+
<build>
46+
<plugins>
47+
<plugin>
48+
<groupId>org.apache.maven.plugins</groupId>
49+
<artifactId>maven-shade-plugin</artifactId>
50+
<executions>
51+
<execution>
52+
<phase>package</phase>
53+
<goals>
54+
<goal>shade</goal>
55+
</goals>
56+
<configuration>
57+
<createDependencyReducedPom>false</createDependencyReducedPom>
58+
<transformers>
59+
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
60+
</transformers>
61+
</configuration>
62+
</execution>
63+
</executions>
64+
</plugin>
65+
</plugins>
66+
</build>
67+
</project>

‎pypaimon/py4j/gateway_server.py‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -102,12 +102,20 @@ def _get_classpath(env):
102102
return os.pathsep.join(classpath)
103103

104104

105+
_HADOOP_DEPS_PACKAGE = 'pypaimon.hadoop-deps'
106+
107+
105108
def _get_hadoop_classpath(env):
106109
if constants.PYPAIMON_HADOOP_CLASSPATH in env:
107110
return env[constants.PYPAIMON_HADOOP_CLASSPATH]
108-
109-
if 'HADOOP_CLASSPATH' in env:
111+
elif 'HADOOP_CLASSPATH' in env:
110112
return env['HADOOP_CLASSPATH']
111113
else:
112-
raise EnvironmentError(f"You haven't set '{constants.PYPAIMON_HADOOP_CLASSPATH}', \
113-
and 'HADOOP_CLASSPATH' is also not set. Ensure one of them is set.")
114+
# use built-in hadoop
115+
jars = importlib.resources.files(_HADOOP_DEPS_PACKAGE)
116+
one_jar = next(iter(jars.iterdir()), None)
117+
if not one_jar:
118+
raise EnvironmentError(f"The built-in Hadoop environment has been broken, this \
119+
is unexpected. You can set one of '{constants.PYPAIMON_HADOOP_CLASSPATH}' or \
120+
'HADOOP_CLASSPATH' to continue.")
121+
return os.path.join(os.path.dirname(str(one_jar)), '*')

‎setup.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,10 +57,12 @@
5757
include_package_data=True,
5858
# releasing tool will generate deps
5959
package_dir={
60-
"pypaimon.jars": "deps/jars"
60+
"pypaimon.jars": "deps/jars",
61+
"pypaimon.hadoop-deps": "deps/hadoop"
6162
},
6263
package_data={
63-
"pypaimon.jars": ["*.jar"]
64+
"pypaimon.jars": ["*.jar"],
65+
"pypaimon.hadoop-deps": ["*.jar"]
6466
},
6567
install_requires=install_requires,
6668
description='Apache Paimon Python API',

‎tools/releasing/create_source_release.sh‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -55,11 +55,13 @@ fi
5555

5656
###########################
5757

58-
# prepare bridge jar
59-
60-
DEPS_DIR=${PROJECT_ROOT}/deps/jars
58+
DEPS_DIR=${PROJECT_ROOT}/deps
6159
rm -rf ${DEPS_DIR}
62-
mkdir -p ${DEPS_DIR}
60+
61+
# prepare bridge jar
62+
BRIDGE_DEPS_DIR=${DEPS_DIR}/jars
63+
mkdir -p ${BRIDGE_DEPS_DIR}
64+
touch ${BRIDGE_DEPS_DIR}/__init__.py
6365

6466
cd ${PROJECT_ROOT}/paimon-python-java-bridge
6567

@@ -70,10 +72,23 @@ if grep -q "<version>.*SNAPSHOT</version>" "pom.xml"; then
7072
fi
7173

7274
# get bridge jar version
73-
JAR_VERSION=$(sed -n 's/.*<version>\(.*\)<\/version>.*/\1/p' pom.xml | head -n 1)
75+
BRIDGE_JAR_VERSION=$(sed -n 's/.*<version>\(.*\)<\/version>.*/\1/p' pom.xml | head -n 1)
76+
77+
mvn clean install -DskipTests
78+
cp "target/paimon-python-java-bridge-${BRIDGE_JAR_VERSION}.jar" ${BRIDGE_DEPS_DIR}
79+
80+
# prepare hadoop-deps jar
81+
HADOOP_DEPS_DIR=${DEPS_DIR}/hadoop
82+
mkdir -p ${HADOOP_DEPS_DIR}
83+
touch ${HADOOP_DEPS_DIR}/__init__.py
84+
85+
cd ${PROJECT_ROOT}/hadoop-deps
86+
87+
# get hadoop-deps jar version
88+
HADOOP_JAR_VERSION=$(sed -n 's/.*<version>\(.*\)<\/version>.*/\1/p' pom.xml | head -n 1)
7489

7590
mvn clean install -DskipTests
76-
cp "target/paimon-python-java-bridge-${JAR_VERSION}.jar" ${DEPS_DIR}
91+
cp "target/hadoop-deps-${HADOOP_JAR_VERSION}.jar" ${HADOOP_DEPS_DIR}
7792

7893
cd ${CURR_DIR}
7994

0 commit comments

Comments
 (0)