Skip to content

Commit c0f0fe0

Browse files
pan3793pan3793
committed
[KYUUBI #7272] Improve build/dependency.sh script
### Why are the changes needed? This PR improves `build/dependency.sh` script inspired by Spark `dev/test-dependencies.sh` Main advantage - the updated script uses `mvn jar:jar jar:test-jar install:install clean -q` to perform a noop install, which is super faster than making a normal `mvn install` because it triggers the dep resolution without compiling the source code. ### How was this patch tested? Pass GHA. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #7272 from pan3793/dep-pr. Closes #7272 5678d47 [Cheng Pan] Improve build/dependency.sh script Authored-by: Cheng Pan <chengpan@apache.org> Signed-off-by: Cheng Pan <chengpan@apache.org>
1 parent 2e79aa2 commit c0f0fe0

2 files changed

Lines changed: 69 additions & 44 deletions

File tree

‎.github/workflows/dep.yml‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -47,15 +47,6 @@ jobs:
4747
cache: 'maven'
4848
- name: Setup Maven
4949
uses: ./.github/actions/setup-maven
50-
- name: Install modules
51-
env:
52-
MAVEN_ARGS: -Dorg.slf4j.simpleLogger.defaultLogLevel=error
53-
run: |
54-
build/mvn clean install \
55-
-Pflink-provided,spark-provided,hive-provided \
56-
-Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip \
57-
-DskipTests \
58-
-pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am
5950
- name: Check dependency list
6051
run: build/dependency.sh
6152
- name: Dependency Review

‎build/dependency.sh‎

Lines changed: 69 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#!/usr/bin/env bash
2+
23
#
34
# Licensed to the Apache Software Foundation (ASF) under one or more
45
# contributor license agreements. See the NOTICE file distributed with
@@ -16,49 +17,51 @@
1617
# limitations under the License.
1718
#
1819

19-
set -o pipefail
20-
set -e
21-
set -x
20+
set -ex
21+
22+
FWDIR="$(cd "`dirname $0`"/..; pwd)"
23+
cd "$FWDIR"
2224

25+
# Explicitly set locale in order to make `sort` output consistent across machines.
26+
# See https://stackoverflow.com/questions/28881 for more details.
2327
export LC_ALL=C
2428

25-
PWD=$(cd "$(dirname "$0")"/.. || exit; pwd)
29+
# Starting with Maven 3.9.0, this variable contains arguments passed to Maven before CLI arguments.
30+
# See https://maven.apache.org/configure.html#maven_args-environment-variable for more details.
31+
export MAVEN_ARGS="-Pflink-provided,spark-provided,hive-provided -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip"
2632

27-
MVN="${PWD}/build/mvn"
33+
MVN="${FWDIR}/build/mvn"
34+
DEP_PR="${FWDIR}/dev/dependencyList.tmp"
35+
DEP="${FWDIR}/dev/dependencyList"
2836

37+
# We'll switch the version to a temp. one, publish POMs using that new version, then switch back to
38+
# the old version. We need to do this because the `dependency:build-classpath` task needs to
39+
# resolve Kyuubi's internal submodule dependencies.
2940

30-
DEP_PR="${PWD}/dev/dependencyList.tmp"
31-
DEP="${PWD}/dev/dependencyList"
41+
# From http://stackoverflow.com/a/26514030
42+
set +e
43+
OLD_VERSION=$(grep -A3 "<artifactId>kyuubi-parent</artifactId>" "${FWDIR}/pom.xml" |
44+
grep "<version>" | head -n1 | awk -F '[<>]' '{print $3}')
45+
set -e
3246

47+
TEMP_VERSION="kyuubi-$(awk 'BEGIN {srand(); print int(100000 + rand() * 900000)}')"
3348

34-
function build_classpath() {
35-
$MVN dependency:build-classpath --no-snapshot-updates -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly |\
36-
grep -v "INFO\|WARN" | \
37-
tail -1 | \
38-
tr ":" "\n" | \
39-
awk -F '/' '{
40-
artifact_id=$(NF-2);
41-
version=$(NF-1);
42-
jar_name=$NF;
43-
classifier_start_index=length(artifact_id"-"version"-") + 1;
44-
classifier_end_index=index(jar_name, ".jar") - 1;
45-
classifier=substr(jar_name, classifier_start_index, classifier_end_index - classifier_start_index + 1);
46-
print artifact_id"/"version"/"classifier"/"jar_name
47-
}' | grep -v "kyuubi" | sort >> "${DEP_PR}"
48-
}
49+
function reset_version {
50+
# Delete the temporary POMs that we wrote to the local Maven repo:
51+
find "$HOME/.m2/" | grep "$TEMP_VERSION" | xargs rm -rf
4952

50-
function check_diff() {
51-
set +e
52-
the_diff=$(diff "${DEP}" "${DEP_PR}")
53-
set -e
54-
rm -rf "${DEP_PR}"
55-
if [[ -n "${the_diff}" ]]; then
56-
echo "Dependency List Changed Detected: "
57-
echo "${the_diff}"
58-
echo "To update the dependency file, run './build/dependency.sh --replace'."
59-
exit 1
60-
fi
53+
# Restore the original version number:
54+
$MVN -q versions:set -DnewVersion=$OLD_VERSION -DgenerateBackupPoms=false > /dev/null
6155
}
56+
trap reset_version EXIT
57+
58+
$MVN -q versions:set -DnewVersion=$TEMP_VERSION -DgenerateBackupPoms=false > /dev/null
59+
60+
echo "Performing Maven install"
61+
$MVN jar:jar jar:test-jar install:install clean -q
62+
63+
echo "Performing Maven validate"
64+
$MVN validate -q
6265

6366
rm -rf "${DEP_PR}"
6467
cat >"${DEP_PR}"<<EOF
@@ -81,12 +84,43 @@ cat >"${DEP_PR}"<<EOF
8184
8285
EOF
8386

84-
build_classpath
87+
echo "Generating dependency manifest"
88+
$MVN dependency:build-classpath -pl kyuubi-assembly -am \
89+
| grep "Dependencies classpath:" -A 1 \
90+
| tail -n 1 | tr ":" "\n" | awk -F '/' '{
91+
# For each dependency classpath, we fetch the last three parts split by "/": artifact id, version, and jar name.
92+
# Since classifier, if exists, always sits between "artifact_id-version-" and ".jar" suffix in the jar name,
93+
# we extract classifier and put it right before the jar name explicitly.
94+
# For example, `orc-core/1.5.5/nohive/orc-core-1.5.5-nohive.jar`
95+
# ^^^^^^
96+
# extracted classifier
97+
# `okio/1.15.0//okio-1.15.0.jar`
98+
# ^
99+
# empty for dependencies without classifier
100+
artifact_id=$(NF-2);
101+
version=$(NF-1);
102+
jar_name=$NF;
103+
classifier_start_index=length(artifact_id"-"version"-") + 1;
104+
classifier_end_index=index(jar_name, ".jar") - 1;
105+
classifier=substr(jar_name, classifier_start_index, classifier_end_index - classifier_start_index + 1);
106+
print artifact_id"/"version"/"classifier"/"jar_name
107+
}' | sort | grep -v kyuubi >> $DEP_PR
85108

86109
if [[ "$1" == "--replace" ]]; then
87110
rm -rf "${DEP}"
88111
mv "${DEP_PR}" "${DEP}"
89112
exit 0
90113
fi
91114

92-
check_diff
115+
set +e
116+
the_diff=$(diff "${DEP}" "${DEP_PR}")
117+
set -e
118+
rm -rf "${DEP_PR}"
119+
if [[ -n "${the_diff}" ]]; then
120+
echo "Dependency List Changed Detected: "
121+
echo "${the_diff}"
122+
echo "To update the dependency file, run './build/dependency.sh --replace'."
123+
exit 1
124+
fi
125+
126+
exit 0

0 commit comments

Comments
 (0)