Skip to content

Commit a15f6e3

Browse files
pwendellpwendell
committed
[SPARK-3996]: Shade Jetty in Spark deliverables
(v2 of this patch with a fix that was only relevant for the maven build). This patch piggy-back's on vanzin's work to simplify the Guava shading, and adds Jetty as a shaded library in Spark. Other than adding Jetty, it consilidates the <artifactSet>'s into the root pom. I found it was a bit easier to follow that way, since you don't need to look into child pom's to find out specific artifact sets included in shading. Author: Patrick Wendell <[email protected]> Closes apache#4285 from pwendell/jetty and squashes the following commits: d3e7f4e [Patrick Wendell] Fix for shaded deps causing compile errors 19f0710 [Patrick Wendell] More code review feedback 961452d [Patrick Wendell] Responding to feedback from Marcello 6df25ca [Patrick Wendell] [WIP] [SPARK-3996]: Shade Jetty in Spark deliverables
1 parent 859f724 commit a15f6e3

File tree

6 files changed

+95
-15
lines changed

6 files changed

+95
-15
lines changed

‎bin/compute-classpath.sh

+3-1
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,8 @@ fi
5050
if [ -n "$SPARK_PREPEND_CLASSES" ]; then
5151
echo "NOTE: SPARK_PREPEND_CLASSES is set, placing locally compiled Spark"\
5252
"classes ahead of assembly." >&2
53+
# Spark classes
5354
CLASSPATH="$CLASSPATH:$FWDIR/core/target/scala-$SPARK_SCALA_VERSION/classes"
54-
CLASSPATH="$CLASSPATH:$FWDIR/core/target/jars/*"
5555
CLASSPATH="$CLASSPATH:$FWDIR/repl/target/scala-$SPARK_SCALA_VERSION/classes"
5656
CLASSPATH="$CLASSPATH:$FWDIR/mllib/target/scala-$SPARK_SCALA_VERSION/classes"
5757
CLASSPATH="$CLASSPATH:$FWDIR/bagel/target/scala-$SPARK_SCALA_VERSION/classes"
@@ -63,6 +63,8 @@ if [ -n "$SPARK_PREPEND_CLASSES" ]; then
6363
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SPARK_SCALA_VERSION/classes"
6464
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive-thriftserver/target/scala-$SPARK_SCALA_VERSION/classes"
6565
CLASSPATH="$CLASSPATH:$FWDIR/yarn/stable/target/scala-$SPARK_SCALA_VERSION/classes"
66+
# Jars for shaded deps in their original form (copied here during build)
67+
CLASSPATH="$CLASSPATH:$FWDIR/core/target/jars/*"
6668
fi
6769

6870
# Use spark-assembly jar from either RELEASE or assembly directory

‎core/pom.xml

+20-2
Original file line numberDiff line numberDiff line change
@@ -94,22 +94,35 @@
9494
<groupId>org.apache.curator</groupId>
9595
<artifactId>curator-recipes</artifactId>
9696
</dependency>
97+
98+
<!-- Jetty dependencies promoted to compile here so they are shaded
99+
and inlined into spark-core jar -->
97100
<dependency>
98101
<groupId>org.eclipse.jetty</groupId>
99102
<artifactId>jetty-plus</artifactId>
103+
<scope>compile</scope>
100104
</dependency>
101105
<dependency>
102106
<groupId>org.eclipse.jetty</groupId>
103107
<artifactId>jetty-security</artifactId>
108+
<scope>compile</scope>
104109
</dependency>
105110
<dependency>
106111
<groupId>org.eclipse.jetty</groupId>
107112
<artifactId>jetty-util</artifactId>
113+
<scope>compile</scope>
108114
</dependency>
109115
<dependency>
110116
<groupId>org.eclipse.jetty</groupId>
111117
<artifactId>jetty-server</artifactId>
118+
<scope>compile</scope>
112119
</dependency>
120+
<dependency>
121+
<groupId>org.eclipse.jetty</groupId>
122+
<artifactId>jetty-http</artifactId>
123+
<scope>compile</scope>
124+
</dependency>
125+
113126
<dependency>
114127
<groupId>org.apache.commons</groupId>
115128
<artifactId>commons-lang3</artifactId>
@@ -348,19 +361,24 @@
348361
<groupId>org.apache.maven.plugins</groupId>
349362
<artifactId>maven-dependency-plugin</artifactId>
350363
<executions>
364+
<!-- When using SPARK_PREPEND_CLASSES Spark classes compiled locally don't use
365+
shaded deps. So here we store jars in their original form which are added
366+
when the classpath is computed. -->
351367
<execution>
352368
<id>copy-dependencies</id>
353369
<phase>package</phase>
354370
<goals>
355371
<goal>copy-dependencies</goal>
356372
</goals>
357-
<configuration>
373+
<configuration>
358374
<outputDirectory>${project.build.directory}</outputDirectory>
359375
<overWriteReleases>false</overWriteReleases>
360376
<overWriteSnapshots>false</overWriteSnapshots>
361377
<overWriteIfNewer>true</overWriteIfNewer>
362378
<useSubDirectoryPerType>true</useSubDirectoryPerType>
363-
<includeArtifactIds>guava</includeArtifactIds>
379+
<includeArtifactIds>
380+
guava,jetty-io,jetty-http,jetty-plus,jetty-util,jetty-server
381+
</includeArtifactIds>
364382
<silent>true</silent>
365383
</configuration>
366384
</execution>

‎network/common/pom.xml

-12
Original file line numberDiff line numberDiff line change
@@ -101,18 +101,6 @@
101101
</execution>
102102
</executions>
103103
</plugin>
104-
<plugin>
105-
<groupId>org.apache.maven.plugins</groupId>
106-
<artifactId>maven-shade-plugin</artifactId>
107-
<configuration>
108-
<shadedArtifactAttached>false</shadedArtifactAttached>
109-
<artifactSet>
110-
<includes>
111-
<include>com.google.guava:guava</include>
112-
</includes>
113-
</artifactSet>
114-
</configuration>
115-
</plugin>
116104
</plugins>
117105
</build>
118106
</project>

‎pom.xml

+32
Original file line numberDiff line numberDiff line change
@@ -337,32 +337,48 @@
337337
</exclusion>
338338
</exclusions>
339339
</dependency>
340+
341+
<!-- Shaded deps marked as provided. These are promoted to compile scope
342+
in the modules where we want the shaded classes to appear in the
343+
associated jar. -->
344+
<dependency>
345+
<groupId>org.eclipse.jetty</groupId>
346+
<artifactId>jetty-http</artifactId>
347+
<version>${jetty.version}</version>
348+
<scope>provided</scope>
349+
</dependency>
340350
<dependency>
341351
<groupId>org.eclipse.jetty</groupId>
342352
<artifactId>jetty-util</artifactId>
343353
<version>${jetty.version}</version>
354+
<scope>provided</scope>
344355
</dependency>
345356
<dependency>
346357
<groupId>org.eclipse.jetty</groupId>
347358
<artifactId>jetty-security</artifactId>
348359
<version>${jetty.version}</version>
360+
<scope>provided</scope>
349361
</dependency>
350362
<dependency>
351363
<groupId>org.eclipse.jetty</groupId>
352364
<artifactId>jetty-plus</artifactId>
353365
<version>${jetty.version}</version>
366+
<scope>provided</scope>
354367
</dependency>
355368
<dependency>
356369
<groupId>org.eclipse.jetty</groupId>
357370
<artifactId>jetty-server</artifactId>
358371
<version>${jetty.version}</version>
372+
<scope>provided</scope>
359373
</dependency>
360374
<dependency>
361375
<groupId>com.google.guava</groupId>
362376
<artifactId>guava</artifactId>
363377
<version>14.0.1</version>
364378
<scope>provided</scope>
365379
</dependency>
380+
<!-- End of shaded deps -->
381+
366382
<dependency>
367383
<groupId>org.apache.commons</groupId>
368384
<artifactId>commons-lang3</artifactId>
@@ -1276,10 +1292,26 @@
12761292
<shadedArtifactAttached>false</shadedArtifactAttached>
12771293
<artifactSet>
12781294
<includes>
1295+
<!-- At a minimum we must include this to force effective pom generation -->
12791296
<include>org.spark-project.spark:unused</include>
1297+
1298+
<include>org.eclipse.jetty:jetty-io</include>
1299+
<include>org.eclipse.jetty:jetty-http</include>
1300+
<include>org.eclipse.jetty:jetty-plus</include>
1301+
<include>org.eclipse.jetty:jetty-security</include>
1302+
<include>org.eclipse.jetty:jetty-util</include>
1303+
<include>org.eclipse.jetty:jetty-server</include>
1304+
<include>com.google.guava:guava</include>
12801305
</includes>
12811306
</artifactSet>
12821307
<relocations>
1308+
<relocation>
1309+
<pattern>org.eclipse.jetty</pattern>
1310+
<shadedPattern>org.spark-project.jetty</shadedPattern>
1311+
<includes>
1312+
<include>org.eclipse.jetty.**</include>
1313+
</includes>
1314+
</relocation>
12831315
<relocation>
12841316
<pattern>com.google.common</pattern>
12851317
<shadedPattern>org.spark-project.guava</shadedPattern>

‎repl/pom.xml

+24
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,30 @@
8787
<artifactId>scalacheck_${scala.binary.version}</artifactId>
8888
<scope>test</scope>
8989
</dependency>
90+
91+
<!-- Explicit listing of transitive deps that are shaded. Otherwise, odd compiler crashes. -->
92+
<dependency>
93+
<groupId>org.eclipse.jetty</groupId>
94+
<artifactId>jetty-server</artifactId>
95+
</dependency>
96+
<dependency>
97+
<groupId>org.eclipse.jetty</groupId>
98+
<artifactId>jetty-plus</artifactId>
99+
</dependency>
100+
<dependency>
101+
<groupId>org.eclipse.jetty</groupId>
102+
<artifactId>jetty-util</artifactId>
103+
</dependency>
104+
<dependency>
105+
<groupId>org.eclipse.jetty</groupId>
106+
<artifactId>jetty-http</artifactId>
107+
</dependency>
108+
<!-- End of shaded deps. -->
109+
110+
<dependency>
111+
<groupId>org.scala-lang</groupId>
112+
<artifactId>scala-library</artifactId>
113+
</dependency>
90114
</dependencies>
91115
<build>
92116
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>

‎streaming/pom.xml

+16
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@
4040
<artifactId>spark-core_${scala.binary.version}</artifactId>
4141
<version>${project.version}</version>
4242
</dependency>
43+
44+
<!-- Explicit listing of transitive deps that are shaded. Otherwise, odd compiler crashes. -->
4345
<dependency>
4446
<groupId>com.google.guava</groupId>
4547
<artifactId>guava</artifactId>
@@ -48,6 +50,20 @@
4850
<groupId>org.eclipse.jetty</groupId>
4951
<artifactId>jetty-server</artifactId>
5052
</dependency>
53+
<dependency>
54+
<groupId>org.eclipse.jetty</groupId>
55+
<artifactId>jetty-plus</artifactId>
56+
</dependency>
57+
<dependency>
58+
<groupId>org.eclipse.jetty</groupId>
59+
<artifactId>jetty-util</artifactId>
60+
</dependency>
61+
<dependency>
62+
<groupId>org.eclipse.jetty</groupId>
63+
<artifactId>jetty-http</artifactId>
64+
</dependency>
65+
<!-- End of shaded deps. -->
66+
5167
<dependency>
5268
<groupId>org.scala-lang</groupId>
5369
<artifactId>scala-library</artifactId>

0 commit comments

Comments
 (0)