use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric in project hadoop by apache.
the class TestTimelineReaderWebServicesHBaseStorage method loadData.
private static void loadData() throws Exception {
String cluster = "cluster1";
String user = "user1";
String flow = "flow_name";
String flowVersion = "CF7022C10F1354";
Long runid = 1002345678919L;
Long runid1 = 1002345678920L;
TimelineEntities te = new TimelineEntities();
TimelineEntity entity = new TimelineEntity();
String id = "application_1111111111_1111";
String type = TimelineEntityType.YARN_APPLICATION.toString();
entity.setId(id);
entity.setType(type);
Long cTime = 1425016501000L;
entity.setCreatedTime(cTime);
entity.addConfig("cfg2", "value1");
// add metrics
Set<TimelineMetric> metrics = new HashSet<>();
TimelineMetric m1 = new TimelineMetric();
m1.setId("MAP_SLOT_MILLIS");
Map<Long, Number> metricValues = ImmutableMap.of(ts - 100000, (Number) 2, ts - 90000, 7, ts - 80000, 40);
m1.setType(Type.TIME_SERIES);
m1.setValues(metricValues);
metrics.add(m1);
m1 = new TimelineMetric();
m1.setId("MAP1_SLOT_MILLIS");
metricValues = ImmutableMap.of(ts - 100000, (Number) 2, ts - 90000, 9, ts - 80000, 40);
m1.setType(Type.TIME_SERIES);
m1.setValues(metricValues);
metrics.add(m1);
m1 = new TimelineMetric();
m1.setId("HDFS_BYTES_READ");
metricValues = ImmutableMap.of(ts - 100000, (Number) 31, ts - 80000, 57);
m1.setType(Type.TIME_SERIES);
m1.setValues(metricValues);
metrics.add(m1);
entity.addMetrics(metrics);
TimelineEvent event = new TimelineEvent();
event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
event.setTimestamp(cTime);
String expKey = "foo_event";
Object expVal = "test";
event.addInfo(expKey, expVal);
entity.addEvent(event);
TimelineEvent event11 = new TimelineEvent();
event11.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
Long expTs = 1425019501000L;
event11.setTimestamp(expTs);
entity.addEvent(event11);
te.addEntity(entity);
// write another application with same metric to this flow
TimelineEntities te1 = new TimelineEntities();
TimelineEntity entity1 = new TimelineEntity();
id = "application_1111111111_2222";
type = TimelineEntityType.YARN_APPLICATION.toString();
entity1.setId(id);
entity1.setType(type);
cTime = 1425016501000L;
entity1.setCreatedTime(cTime);
entity1.addConfig("cfg1", "value1");
// add metrics
metrics.clear();
TimelineMetric m2 = new TimelineMetric();
m2.setId("MAP_SLOT_MILLIS");
metricValues = new HashMap<Long, Number>();
metricValues.put(ts - 100000, 5L);
metricValues.put(ts - 80000, 101L);
m2.setType(Type.TIME_SERIES);
m2.setValues(metricValues);
metrics.add(m2);
entity1.addMetrics(metrics);
TimelineEvent event1 = new TimelineEvent();
event1.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
event1.setTimestamp(cTime);
event1.addInfo(expKey, expVal);
entity1.addEvent(event1);
te1.addEntity(entity1);
String flow2 = "flow_name2";
String flowVersion2 = "CF7022C10F1454";
Long runid2 = 2102356789046L;
TimelineEntities te3 = new TimelineEntities();
TimelineEntity entity3 = new TimelineEntity();
id = "application_11111111111111_2223";
entity3.setId(id);
entity3.setType(type);
cTime = 1425016501037L;
entity3.setCreatedTime(cTime);
TimelineEvent event2 = new TimelineEvent();
event2.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
event2.setTimestamp(cTime);
event2.addInfo("foo_event", "test");
entity3.addEvent(event2);
te3.addEntity(entity3);
TimelineEntities te4 = new TimelineEntities();
TimelineEntity entity4 = new TimelineEntity();
id = "application_1111111111_2224";
entity4.setId(id);
entity4.setType(type);
cTime = 1425016501034L;
entity4.setCreatedTime(cTime);
TimelineEvent event4 = new TimelineEvent();
event4.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
event4.setTimestamp(cTime);
event4.addInfo("foo_event", "test");
entity4.addEvent(event4);
metrics.clear();
m2 = new TimelineMetric();
m2.setId("MAP_SLOT_MILLIS");
metricValues = ImmutableMap.of(ts - 100000, (Number) 5L, ts - 80000, 101L);
m2.setType(Type.TIME_SERIES);
m2.setValues(metricValues);
metrics.add(m2);
entity4.addMetrics(metrics);
te4.addEntity(entity4);
TimelineEntities te5 = new TimelineEntities();
TimelineEntity entity5 = new TimelineEntity();
entity5.setId("entity1");
entity5.setType("type1");
entity5.setCreatedTime(1425016501034L);
// add some config entries
entity5.addConfigs(ImmutableMap.of("config_param1", "value1", "config_param2", "value2", "cfg_param1", "value3"));
entity5.addInfo(ImmutableMap.of("info1", (Object) "cluster1", "info2", 2.0, "info3", 35000, "info4", 36000));
metrics = new HashSet<>();
m1 = new TimelineMetric();
m1.setId("MAP_SLOT_MILLIS");
metricValues = ImmutableMap.of(ts - 100000, (Number) 2, ts - 80000, 40);
m1.setType(Type.TIME_SERIES);
m1.setValues(metricValues);
metrics.add(m1);
m1 = new TimelineMetric();
m1.setId("HDFS_BYTES_READ");
metricValues = ImmutableMap.of(ts - 100000, (Number) 31, ts - 80000, 57);
m1.setType(Type.TIME_SERIES);
m1.setValues(metricValues);
metrics.add(m1);
entity5.addMetrics(metrics);
TimelineEvent event51 = new TimelineEvent();
event51.setId("event1");
event51.setTimestamp(cTime);
entity5.addEvent(event51);
TimelineEvent event52 = new TimelineEvent();
event52.setId("event2");
event52.setTimestamp(cTime);
entity5.addEvent(event52);
TimelineEvent event53 = new TimelineEvent();
event53.setId("event3");
event53.setTimestamp(cTime);
entity5.addEvent(event53);
TimelineEvent event54 = new TimelineEvent();
event54.setId("event4");
event54.setTimestamp(cTime);
entity5.addEvent(event54);
Map<String, Set<String>> isRelatedTo1 = new HashMap<String, Set<String>>();
isRelatedTo1.put("type2", Sets.newHashSet("entity21", "entity22", "entity23", "entity24"));
isRelatedTo1.put("type4", Sets.newHashSet("entity41", "entity42"));
isRelatedTo1.put("type1", Sets.newHashSet("entity14", "entity15"));
isRelatedTo1.put("type3", Sets.newHashSet("entity31", "entity35", "entity32", "entity33"));
entity5.addIsRelatedToEntities(isRelatedTo1);
Map<String, Set<String>> relatesTo1 = new HashMap<String, Set<String>>();
relatesTo1.put("type2", Sets.newHashSet("entity21", "entity22", "entity23", "entity24"));
relatesTo1.put("type4", Sets.newHashSet("entity41", "entity42"));
relatesTo1.put("type1", Sets.newHashSet("entity14", "entity15"));
relatesTo1.put("type3", Sets.newHashSet("entity31", "entity35", "entity32", "entity33"));
entity5.addRelatesToEntities(relatesTo1);
te5.addEntity(entity5);
TimelineEntity entity6 = new TimelineEntity();
entity6.setId("entity2");
entity6.setType("type1");
entity6.setCreatedTime(1425016501034L);
entity6.addConfigs(ImmutableMap.of("cfg_param3", "value1", "configuration_param2", "value2", "config_param1", "value3"));
entity6.addInfo(ImmutableMap.of("info1", (Object) "cluster2", "info2", 2.0, "info4", 35000));
metrics = new HashSet<>();
m1 = new TimelineMetric();
m1.setId("MAP1_SLOT_MILLIS");
metricValues = ImmutableMap.of(ts - 100000, (Number) 12, ts - 80000, 140);
m1.setType(Type.TIME_SERIES);
m1.setValues(metricValues);
metrics.add(m1);
m1 = new TimelineMetric();
m1.setId("HDFS_BYTES_READ");
metricValues = ImmutableMap.of(ts - 100000, (Number) 78, ts - 80000, 157);
m1.setType(Type.TIME_SERIES);
m1.setValues(metricValues);
metrics.add(m1);
m1 = new TimelineMetric();
m1.setId("MAP11_SLOT_MILLIS");
m1.setType(Type.SINGLE_VALUE);
m1.addValue(ts - 100000, 122);
metrics.add(m1);
entity6.addMetrics(metrics);
TimelineEvent event61 = new TimelineEvent();
event61.setId("event1");
event61.setTimestamp(cTime);
entity6.addEvent(event61);
TimelineEvent event62 = new TimelineEvent();
event62.setId("event5");
event62.setTimestamp(cTime);
entity6.addEvent(event62);
TimelineEvent event63 = new TimelineEvent();
event63.setId("event3");
event63.setTimestamp(cTime);
entity6.addEvent(event63);
TimelineEvent event64 = new TimelineEvent();
event64.setId("event6");
event64.setTimestamp(cTime);
entity6.addEvent(event64);
Map<String, Set<String>> isRelatedTo2 = new HashMap<String, Set<String>>();
isRelatedTo2.put("type2", Sets.newHashSet("entity21", "entity22", "entity23", "entity24"));
isRelatedTo2.put("type5", Sets.newHashSet("entity51", "entity52"));
isRelatedTo2.put("type6", Sets.newHashSet("entity61", "entity66"));
isRelatedTo2.put("type3", Sets.newHashSet("entity31"));
entity6.addIsRelatedToEntities(isRelatedTo2);
Map<String, Set<String>> relatesTo2 = new HashMap<String, Set<String>>();
relatesTo2.put("type2", Sets.newHashSet("entity21", "entity22", "entity23", "entity24"));
relatesTo2.put("type5", Sets.newHashSet("entity51", "entity52"));
relatesTo2.put("type6", Sets.newHashSet("entity61", "entity66"));
relatesTo2.put("type3", Sets.newHashSet("entity31"));
entity6.addRelatesToEntities(relatesTo2);
te5.addEntity(entity6);
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
hbi.write(cluster, user, flow, flowVersion, runid, entity.getId(), te);
hbi.write(cluster, user, flow, flowVersion, runid, entity1.getId(), te1);
hbi.write(cluster, user, flow, flowVersion, runid1, entity4.getId(), te4);
hbi.write(cluster, user, flow2, flowVersion2, runid2, entity3.getId(), te3);
hbi.write(cluster, user, flow, flowVersion, runid, "application_1111111111_1111", te5);
hbi.flush();
} finally {
if (hbi != null) {
hbi.close();
}
}
}
use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric in project hadoop by apache.
the class HBaseTimelineWriterImpl method storeMetrics.
/**
* stores the {@linkplain TimelineMetric} information from the
* {@linkplain TimelineEvent} object.
*/
private void storeMetrics(byte[] rowKey, Set<TimelineMetric> metrics, boolean isApplication) throws IOException {
if (metrics != null) {
for (TimelineMetric metric : metrics) {
byte[] metricColumnQualifier = stringKeyConverter.encode(metric.getId());
Map<Long, Number> timeseries = metric.getValues();
for (Map.Entry<Long, Number> timeseriesEntry : timeseries.entrySet()) {
Long timestamp = timeseriesEntry.getKey();
if (isApplication) {
ApplicationColumnPrefix.METRIC.store(rowKey, applicationTable, metricColumnQualifier, timestamp, timeseriesEntry.getValue());
} else {
EntityColumnPrefix.METRIC.store(rowKey, entityTable, metricColumnQualifier, timestamp, timeseriesEntry.getValue());
}
}
}
}
}
use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric in project hadoop by apache.
the class TestTimelineReaderWebServicesHBaseStorage method testGetFlowRun.
@Test
public void testGetFlowRun() throws Exception {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs/" + "1002345678919");
ClientResponse resp = getResponse(client, uri);
FlowRunEntity entity = resp.getEntity(FlowRunEntity.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; charset=utf-8", resp.getType().toString());
assertNotNull(entity);
assertEquals("user1@flow_name/1002345678919", entity.getId());
assertEquals(3, entity.getMetrics().size());
TimelineMetric m1 = newMetric(TimelineMetric.Type.SINGLE_VALUE, "HDFS_BYTES_READ", ts - 80000, 57L);
TimelineMetric m2 = newMetric(TimelineMetric.Type.SINGLE_VALUE, "MAP_SLOT_MILLIS", ts - 80000, 141L);
TimelineMetric m3 = newMetric(TimelineMetric.Type.SINGLE_VALUE, "MAP1_SLOT_MILLIS", ts - 80000, 40L);
for (TimelineMetric metric : entity.getMetrics()) {
assertTrue(verifyMetrics(metric, m1, m2, m3));
}
// Query without specifying cluster ID.
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/users/user1/flows/flow_name/runs/1002345678919");
resp = getResponse(client, uri);
entity = resp.getEntity(FlowRunEntity.class);
assertNotNull(entity);
assertEquals("user1@flow_name/1002345678919", entity.getId());
assertEquals(3, entity.getMetrics().size());
m1 = newMetric(TimelineMetric.Type.SINGLE_VALUE, "HDFS_BYTES_READ", ts - 80000, 57L);
m2 = newMetric(TimelineMetric.Type.SINGLE_VALUE, "MAP_SLOT_MILLIS", ts - 80000, 141L);
m3 = newMetric(TimelineMetric.Type.SINGLE_VALUE, "MAP1_SLOT_MILLIS", ts - 80000, 40L);
for (TimelineMetric metric : entity.getMetrics()) {
assertTrue(verifyMetrics(metric, m1, m2, m3));
}
} finally {
client.destroy();
}
}
use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric in project hadoop by apache.
the class TestTimelineReaderWebServicesHBaseStorage method testGetEntitiesDataToRetrieve.
/**
* Tests if specific configs and metrics are retrieve for getEntities call.
*/
@Test
public void testGetEntitiesDataToRetrieve() throws Exception {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?confstoretrieve=cfg_");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
});
assertNotNull(entities);
assertEquals(2, entities.size());
int cfgCnt = 0;
for (TimelineEntity entity : entities) {
cfgCnt += entity.getConfigs().size();
for (String configKey : entity.getConfigs().keySet()) {
assertTrue(configKey.startsWith("cfg_"));
}
}
assertEquals(2, cfgCnt);
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?confstoretrieve=cfg_,config_");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
});
assertNotNull(entities);
assertEquals(2, entities.size());
cfgCnt = 0;
for (TimelineEntity entity : entities) {
cfgCnt += entity.getConfigs().size();
for (String configKey : entity.getConfigs().keySet()) {
assertTrue(configKey.startsWith("cfg_") || configKey.startsWith("config_"));
}
}
assertEquals(5, cfgCnt);
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?confstoretrieve=!(cfg_,config_)");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
});
assertNotNull(entities);
assertEquals(2, entities.size());
cfgCnt = 0;
for (TimelineEntity entity : entities) {
cfgCnt += entity.getConfigs().size();
for (String configKey : entity.getConfigs().keySet()) {
assertTrue(configKey.startsWith("configuration_"));
}
}
assertEquals(1, cfgCnt);
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?metricstoretrieve=MAP_");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
});
assertNotNull(entities);
assertEquals(2, entities.size());
int metricCnt = 0;
for (TimelineEntity entity : entities) {
metricCnt += entity.getMetrics().size();
for (TimelineMetric metric : entity.getMetrics()) {
assertTrue(metric.getId().startsWith("MAP_"));
}
}
assertEquals(1, metricCnt);
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?metricstoretrieve=MAP1_,HDFS_");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
});
assertNotNull(entities);
assertEquals(2, entities.size());
metricCnt = 0;
for (TimelineEntity entity : entities) {
metricCnt += entity.getMetrics().size();
for (TimelineMetric metric : entity.getMetrics()) {
assertTrue(metric.getId().startsWith("MAP1_") || metric.getId().startsWith("HDFS_"));
}
}
assertEquals(3, metricCnt);
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?metricstoretrieve=!(MAP1_,HDFS_)");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
});
assertNotNull(entities);
assertEquals(2, entities.size());
metricCnt = 0;
for (TimelineEntity entity : entities) {
metricCnt += entity.getMetrics().size();
for (TimelineMetric metric : entity.getMetrics()) {
assertTrue(metric.getId().startsWith("MAP_") || metric.getId().startsWith("MAP11_"));
}
}
assertEquals(2, metricCnt);
} finally {
client.destroy();
}
}
use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric in project hadoop by apache.
the class TestTimelineReaderWebServicesHBaseStorage method testGetFlowApps.
@Test
public void testGetFlowApps() throws Exception {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/apps?" + "fields=ALL");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
});
assertNotNull(entities);
assertEquals(3, entities.size());
for (TimelineEntity entity : entities) {
assertTrue("Unexpected app in result", (entity.getId().equals("application_1111111111_1111") && entity.getConfigs().size() == 1 && entity.getConfigs().equals(ImmutableMap.of("cfg2", "value1"))) || (entity.getId().equals("application_1111111111_2222") && entity.getConfigs().size() == 1 && entity.getConfigs().equals(ImmutableMap.of("cfg1", "value1"))) || (entity.getId().equals("application_1111111111_2224") && entity.getConfigs().size() == 0));
for (TimelineMetric metric : entity.getMetrics()) {
if (entity.getId().equals("application_1111111111_1111")) {
TimelineMetric m1 = newMetric(TimelineMetric.Type.SINGLE_VALUE, "HDFS_BYTES_READ", ts - 80000, 57L);
TimelineMetric m2 = newMetric(TimelineMetric.Type.SINGLE_VALUE, "MAP_SLOT_MILLIS", ts - 80000, 40L);
TimelineMetric m3 = newMetric(TimelineMetric.Type.SINGLE_VALUE, "MAP1_SLOT_MILLIS", ts - 80000, 40L);
assertTrue(verifyMetrics(metric, m1, m2, m3));
} else if (entity.getId().equals("application_1111111111_2222")) {
TimelineMetric m1 = newMetric(TimelineMetric.Type.SINGLE_VALUE, "MAP_SLOT_MILLIS", ts - 80000, 101L);
assertTrue(verifyMetrics(metric, m1));
} else if (entity.getId().equals("application_1111111111_2224")) {
TimelineMetric m1 = newMetric(TimelineMetric.Type.SINGLE_VALUE, "MAP_SLOT_MILLIS", ts - 80000, 101L);
assertTrue(verifyMetrics(metric, m1));
}
}
}
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/apps?" + "fields=ALL&metricslimit=6");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
});
assertNotNull(entities);
assertEquals(3, entities.size());
for (TimelineEntity entity : entities) {
assertTrue("Unexpected app in result", (entity.getId().equals("application_1111111111_1111") && entity.getConfigs().size() == 1 && entity.getConfigs().equals(ImmutableMap.of("cfg2", "value1"))) || (entity.getId().equals("application_1111111111_2222") && entity.getConfigs().size() == 1 && entity.getConfigs().equals(ImmutableMap.of("cfg1", "value1"))) || (entity.getId().equals("application_1111111111_2224") && entity.getConfigs().size() == 0));
for (TimelineMetric metric : entity.getMetrics()) {
if (entity.getId().equals("application_1111111111_1111")) {
TimelineMetric m1 = newMetric(TimelineMetric.Type.TIME_SERIES, "HDFS_BYTES_READ", ts - 80000, 57L);
m1.addValue(ts - 100000, 31L);
TimelineMetric m2 = newMetric(TimelineMetric.Type.TIME_SERIES, "MAP_SLOT_MILLIS", ts - 80000, 40L);
m2.addValue(ts - 100000, 2L);
TimelineMetric m3 = newMetric(TimelineMetric.Type.TIME_SERIES, "MAP1_SLOT_MILLIS", ts - 80000, 40L);
m3.addValue(ts - 100000, 2L);
assertTrue(verifyMetrics(metric, m1, m2, m3));
} else if (entity.getId().equals("application_1111111111_2222")) {
TimelineMetric m1 = newMetric(TimelineMetric.Type.TIME_SERIES, "MAP_SLOT_MILLIS", ts - 80000, 101L);
m1.addValue(ts - 100000, 5L);
assertTrue(verifyMetrics(metric, m1));
} else if (entity.getId().equals("application_1111111111_2224")) {
TimelineMetric m1 = newMetric(TimelineMetric.Type.TIME_SERIES, "MAP_SLOT_MILLIS", ts - 80000, 101L);
m1.addValue(ts - 100000, 5L);
assertTrue(verifyMetrics(metric, m1));
}
}
}
// Query without specifying cluster ID.
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/users/user1/flows/flow_name/apps");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
});
assertNotNull(entities);
assertEquals(3, entities.size());
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/users/user1/flows/flow_name/apps?limit=1");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
});
assertNotNull(entities);
assertEquals(1, entities.size());
} finally {
client.destroy();
}
}
Aggregations