Skip to content

Commit 10a4160

Browse files
K0K0V0KK0K0V0K
committed
YARN-11874. FIX YARN REST IF
WIP, but till that here you can read The Charge of the Light Brigade By Alfred, Lord Tennyson: I Half a league, half a league, Half a league onward, All in the valley of Death Rode the six hundred. “Forward, the Light Brigade! Charge for the guns!” he said. Into the valley of Death Rode the six hundred. II “Forward, the Light Brigade!” Was there a man dismayed? Not though the soldier knew Someone had blundered. Theirs not to make reply, Theirs not to reason why, Theirs but to do and die. Into the valley of Death Rode the six hundred. III Cannon to right of them, Cannon to left of them, Cannon in front of them Volleyed and thundered; Stormed at with shot and shell, Boldly they rode and well, Into the jaws of Death, Into the mouth of hell Rode the six hundred. IV Flashed all their sabres bare, Flashed as they turned in air Sabring the gunners there, Charging an army, while All the world wondered. Plunged in the battery-smoke Right through the line they broke; Cossack and Russian Reeled from the sabre stroke Shattered and sundered. Then they rode back, but not Not the six hundred. V Cannon to right of them, Cannon to left of them, Cannon behind them Volleyed and thundered; Stormed at with shot and shell, While horse and hero fell. They that had fought so well Came through the jaws of Death, Back from the mouth of hell, All that was left of them, Left of six hundred. VI When can their glory fade? O the wild charge they made! All the world wondered. Honour the charge they made! Honour the Light Brigade, Noble six hundred!
1 parent 4e1296c commit 10a4160

File tree

91 files changed

+8723
-16520
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

91 files changed

+8723
-16520
lines changed

‎hadoop-client-modules/hadoop-client-minicluster/pom.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -725,6 +725,13 @@
725725
<exclude>**/*.java</exclude>
726726
</excludes>
727727
</filter>
728+
<!-- Some of our dependencies include html, so remove it. -->
729+
<filter>
730+
<artifact>*:*</artifact>
731+
<excludes>
732+
<exclude>**/*.html</exclude>
733+
</excludes>
734+
</filter>
728735
<!-- We pull in several test jars; keep out the actual test classes -->
729736
<filter>
730737
<artifact>*:*</artifact>

‎hadoop-project/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2146,6 +2146,11 @@
21462146
<artifactId>jersey-media-json-jettison</artifactId>
21472147
<version>${jersey2.version}</version>
21482148
</dependency>
2149+
<dependency>
2150+
<groupId>org.glassfish.jersey.media</groupId>
2151+
<artifactId>jersey-media-moxy</artifactId>
2152+
<version>${jersey2.version}</version>
2153+
</dependency>
21492154
<dependency>
21502155
<groupId>org.glassfish.jaxb</groupId>
21512156
<artifactId>jaxb-runtime</artifactId>

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,10 @@
265265
<groupId>org.glassfish.jersey.media</groupId>
266266
<artifactId>jersey-media-json-jettison</artifactId>
267267
</dependency>
268+
<dependency>
269+
<groupId>org.glassfish.jersey.media</groupId>
270+
<artifactId>jersey-media-moxy</artifactId>
271+
</dependency>
268272
<dependency>
269273
<groupId>org.mockito</groupId>
270274
<artifactId>mockito-junit-jupiter</artifactId>

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java‎

Lines changed: 27 additions & 117 deletions
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,18 @@
1818

1919
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
2020

21+
import org.eclipse.persistence.jaxb.JAXBContextFactory;
22+
import org.eclipse.persistence.jaxb.MarshallerProperties;
23+
import org.glassfish.jersey.jettison.JettisonConfig;
2124
import org.glassfish.jersey.jettison.JettisonJaxbContext;
2225
import org.slf4j.Logger;
2326
import org.slf4j.LoggerFactory;
2427

25-
import java.util.Arrays;
26-
import java.util.ArrayList;
2728
import java.util.Collections;
29+
import java.util.List;
2830
import java.util.Map;
2931
import java.util.HashMap;
32+
import java.util.Set;
3033

3134
import javax.inject.Inject;
3235
import javax.inject.Singleton;
@@ -35,138 +38,45 @@
3538
import javax.xml.bind.JAXBContext;
3639

3740
import org.apache.hadoop.conf.Configuration;
38-
import org.apache.hadoop.yarn.conf.YarnConfiguration;
39-
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo;
40-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
41-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
42-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo;
43-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo;
44-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo;
45-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo;
46-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo;
47-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
48-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
49-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.UserMetricsInfo;
50-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
51-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
52-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
53-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
54-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfoList;
55-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
56-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.UsersInfo;
57-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo;
58-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
59-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerHealthInfo;
60-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerQueueInfoList;
61-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutsInfo;
62-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutInfo;
63-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInformationsInfo;
64-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ActivitiesInfo;
65-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppActivitiesInfo;
66-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueAclsInfo;
67-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueAclInfo;
68-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesInfo;
69-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewApplication;
70-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo;
71-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ContainerLaunchContextInfo;
72-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo;
73-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.DelegationToken;
74-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue;
75-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority;
76-
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo;
77-
import org.apache.hadoop.yarn.webapp.RemoteExceptionData;
41+
import org.apache.hadoop.yarn.server.resourcemanager.webapp.jsonprovider.ClassSerialisationConfig;
7842

7943
@Singleton
8044
@Provider
8145
public class JAXBContextResolver implements ContextResolver<JAXBContext> {
8246

8347
private static final Logger LOG = LoggerFactory.getLogger(JAXBContextResolver.class.getName());
8448

85-
private final Map<Class, JAXBContext> typesContextMap;
49+
private final Map<Class, JAXBContext> typesContextMap = new HashMap<>();
8650

8751
public JAXBContextResolver() throws Exception {
8852
this(new Configuration());
8953
}
9054

9155
@Inject
9256
public JAXBContextResolver(@javax.inject.Named("conf") Configuration conf) throws Exception {
93-
94-
JAXBContext context;
95-
JAXBContext unWrappedRootContext;
96-
97-
// you have to specify all the dao classes here
98-
final Class[] cTypes =
99-
{ AppInfo.class, AppAttemptInfo.class, AppAttemptsInfo.class,
100-
ClusterInfo.class, CapacitySchedulerQueueInfo.class,
101-
FifoSchedulerInfo.class, SchedulerTypeInfo.class, NodeInfo.class,
102-
UserMetricsInfo.class, CapacitySchedulerInfo.class,
103-
ClusterMetricsInfo.class, SchedulerInfo.class, AppsInfo.class,
104-
NodesInfo.class, RemoteExceptionData.class,
105-
CapacitySchedulerQueueInfoList.class, ResourceInfo.class,
106-
UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class,
107-
StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class,
108-
FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class,
109-
AppTimeoutInfo.class, ResourceInformationsInfo.class,
110-
ActivitiesInfo.class, AppActivitiesInfo.class,
111-
QueueAclsInfo.class, QueueAclInfo.class,
112-
BulkActivitiesInfo.class};
113-
114-
// these dao classes need root unwrapping
115-
final Class[] rootUnwrappedTypes =
116-
{ NewApplication.class, ApplicationSubmissionContextInfo.class,
117-
ContainerLaunchContextInfo.class, LocalResourceInfo.class,
118-
DelegationToken.class, AppQueue.class, AppPriority.class,
119-
ResourceOptionInfo.class };
120-
121-
ArrayList<Class> finalcTypesList = new ArrayList<>();
122-
ArrayList<Class> finalRootUnwrappedTypesList = new ArrayList<>();
123-
124-
Collections.addAll(finalcTypesList, cTypes);
125-
Collections.addAll(finalRootUnwrappedTypesList, rootUnwrappedTypes);
126-
127-
// Add Custom DAO Classes
128-
Class[] daoClasses = null;
129-
Class[] unwrappedDaoClasses = null;
130-
boolean loadCustom = true;
131-
try {
132-
daoClasses = conf
133-
.getClasses(YarnConfiguration.YARN_HTTP_WEBAPP_CUSTOM_DAO_CLASSES);
134-
unwrappedDaoClasses = conf.getClasses(
135-
YarnConfiguration.YARN_HTTP_WEBAPP_CUSTOM_UNWRAPPED_DAO_CLASSES);
136-
} catch (Exception e) {
137-
LOG.warn("Failed to load custom dao class: ", e);
138-
loadCustom = false;
139-
}
140-
141-
if (loadCustom) {
142-
if (daoClasses != null) {
143-
Collections.addAll(finalcTypesList, daoClasses);
144-
LOG.debug("Added custom dao classes: {}.", Arrays.toString(daoClasses));
145-
}
146-
if (unwrappedDaoClasses != null) {
147-
Collections.addAll(finalRootUnwrappedTypesList, unwrappedDaoClasses);
148-
LOG.debug("Added custom Unwrapped dao classes: {}", Arrays.toString(unwrappedDaoClasses));
149-
}
150-
}
151-
152-
final Class[] finalcTypes = finalcTypesList
153-
.toArray(new Class[finalcTypesList.size()]);
154-
final Class[] finalRootUnwrappedTypes = finalRootUnwrappedTypesList
155-
.toArray(new Class[finalRootUnwrappedTypesList.size()]);
156-
157-
this.typesContextMap = new HashMap<>();
158-
context = new JettisonJaxbContext(finalcTypes);
159-
unWrappedRootContext = new JettisonJaxbContext(finalRootUnwrappedTypes);
160-
for (Class type : finalcTypes) {
161-
typesContextMap.put(type, context);
162-
}
163-
for (Class type : finalRootUnwrappedTypes) {
164-
typesContextMap.put(type, unWrappedRootContext);
165-
}
57+
ClassSerialisationConfig classSerialisationConfig = new ClassSerialisationConfig(conf);
58+
Set<Class<?>> wrappedClasses = classSerialisationConfig.getWrappedClasses();
59+
Set<Class<?>> unWrappedClasses = classSerialisationConfig.getUnWrappedClasses();
60+
61+
//WARNING: AFAIK these properties not respected by MOXyJsonProvider
62+
//For details check MOXyJsonProvider#readFrom method
63+
JAXBContext wrappedContext = JAXBContextFactory.createContext(
64+
wrappedClasses.toArray(new Class[0]),
65+
Collections.singletonMap(MarshallerProperties.JSON_INCLUDE_ROOT, true)
66+
);
67+
JAXBContext unWrappedContext = JAXBContextFactory.createContext(
68+
unWrappedClasses.toArray(new Class[0]),
69+
Collections.singletonMap(MarshallerProperties.JSON_INCLUDE_ROOT, false)
70+
);
71+
72+
wrappedClasses.forEach(type -> typesContextMap.put(type, wrappedContext));
73+
unWrappedClasses.forEach(type -> typesContextMap.put(type, unWrappedContext));
16674
}
16775

16876
@Override
16977
public JAXBContext getContext(Class<?> objectType) {
170-
return typesContextMap.get(objectType);
78+
JAXBContext jaxbContext = typesContextMap.get(objectType);
79+
LOG.trace("Context for {} is {}", objectType, jaxbContext);
80+
return jaxbContext;
17181
}
17282
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,13 @@
2323
import java.net.InetSocketAddress;
2424

2525
import org.apache.hadoop.conf.Configuration;
26+
27+
import org.glassfish.jersey.moxy.json.MoxyJsonConfig;
2628
import org.slf4j.Logger;
2729
import org.slf4j.LoggerFactory;
2830
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
2931
import org.apache.hadoop.yarn.conf.YarnConfiguration;
32+
import org.apache.hadoop.yarn.server.resourcemanager.webapp.jsonprovider.JsonProviderFeature;
3033
import org.apache.hadoop.yarn.util.RMHAUtils;
3134
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
3235
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
@@ -60,7 +63,8 @@ public ResourceConfig resourceConfig() {
6063
config.register(new JerseyBinder());
6164
config.register(RMWebServices.class);
6265
config.register(GenericExceptionHandler.class);
63-
config.register(new JettisonFeature()).register(JAXBContextResolver.class);
66+
config.register(JsonProviderFeature.class);
67+
config.register(JAXBContextResolver.class);
6468
return config;
6569
}
6670

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -32,59 +32,63 @@
3232
public class NodeLabelsInfo {
3333

3434
@XmlElement(name = "nodeLabelInfo")
35-
private ArrayList<NodeLabelInfo> nodeLabelsInfo = new ArrayList<>();
35+
private ArrayList<NodeLabelInfo> nodeLabelInfo = new ArrayList<>();
3636

3737
public NodeLabelsInfo() {
3838
// JAXB needs this
3939
}
4040

4141
public NodeLabelsInfo(ArrayList<NodeLabelInfo> nodeLabels) {
42-
this.nodeLabelsInfo = nodeLabels;
42+
this.nodeLabelInfo = nodeLabels;
4343
}
4444

4545
public NodeLabelsInfo(List<NodeLabel> nodeLabels) {
46-
this.nodeLabelsInfo = new ArrayList<>();
46+
this.nodeLabelInfo = new ArrayList<>();
4747
for (NodeLabel label : nodeLabels) {
48-
this.nodeLabelsInfo.add(new NodeLabelInfo(label));
48+
this.nodeLabelInfo.add(new NodeLabelInfo(label));
4949
}
5050
}
51-
51+
5252
public NodeLabelsInfo(Set<String> nodeLabelsName) {
53-
this.nodeLabelsInfo = new ArrayList<>();
53+
this.nodeLabelInfo = new ArrayList<>();
5454
for (String labelName : nodeLabelsName) {
55-
this.nodeLabelsInfo.add(new NodeLabelInfo(labelName));
55+
this.nodeLabelInfo.add(new NodeLabelInfo(labelName));
5656
}
5757
}
5858

5959
public NodeLabelsInfo(Collection<NodeLabel> nodeLabels) {
60-
this.nodeLabelsInfo = new ArrayList<>();
60+
this.nodeLabelInfo = new ArrayList<>();
6161
nodeLabels.stream().forEach(nodeLabel -> {
62-
this.nodeLabelsInfo.add(new NodeLabelInfo(nodeLabel));
62+
this.nodeLabelInfo.add(new NodeLabelInfo(nodeLabel));
6363
});
6464
}
6565

6666
public ArrayList<NodeLabelInfo> getNodeLabelsInfo() {
67-
return nodeLabelsInfo;
67+
return nodeLabelInfo;
6868
}
6969

7070
public Set<NodeLabel> getNodeLabels() {
7171
Set<NodeLabel> nodeLabels = new HashSet<>();
72-
for (NodeLabelInfo label : nodeLabelsInfo) {
72+
for (NodeLabelInfo label : nodeLabelInfo) {
7373
nodeLabels.add(NodeLabel.newInstance(label.getName(),
7474
label.getExclusivity()));
7575
}
7676
return nodeLabels;
7777
}
78-
78+
7979
public List<String> getNodeLabelsName() {
8080
ArrayList<String> nodeLabelsName = new ArrayList<>();
81-
for (NodeLabelInfo label : nodeLabelsInfo) {
81+
for (NodeLabelInfo label : nodeLabelInfo) {
8282
nodeLabelsName.add(label.getName());
8383
}
8484
return nodeLabelsName;
8585
}
8686

8787
public void setNodeLabelsInfo(ArrayList<NodeLabelInfo> nodeLabelInfo) {
88-
this.nodeLabelsInfo = nodeLabelInfo;
88+
this.nodeLabelInfo = nodeLabelInfo;
89+
}
90+
91+
public void setNodeLabelInfo(ArrayList<NodeLabelInfo> nodeLabelsInfo) {
92+
this.nodeLabelInfo = nodeLabelsInfo;
8993
}
9094
}

0 commit comments

Comments
 (0)