Search in sources :

Example 6 with TimelineMetric

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric in project hadoop by apache.

the class TestFlowDataGenerator method getEntityMetricsApp2.

static TimelineEntity getEntityMetricsApp2(long insertTs) {
    TimelineEntity entity = new TimelineEntity();
    String id = "flowRunMetrics_test";
    String type = TimelineEntityType.YARN_APPLICATION.toString();
    entity.setId(id);
    entity.setType(type);
    long cTime = 1425016501000L;
    entity.setCreatedTime(cTime);
    // add metrics
    Set<TimelineMetric> metrics = new HashSet<>();
    TimelineMetric m1 = new TimelineMetric();
    m1.setId(METRIC_1);
    Map<Long, Number> metricValues = new HashMap<Long, Number>();
    long ts = insertTs;
    metricValues.put(ts - 100000, 5L);
    metricValues.put(ts - 80000, 101L);
    m1.setType(Type.TIME_SERIES);
    m1.setValues(metricValues);
    metrics.add(m1);
    entity.addMetrics(metrics);
    TimelineEvent event = new TimelineEvent();
    event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
    long endTs = 1439379885000L;
    event.setTimestamp(endTs);
    String expKey = "foo_event_greater";
    String expVal = "test_app_greater";
    event.addInfo(expKey, expVal);
    entity.addEvent(event);
    return entity;
}
Also used : TimelineEvent(org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent) TimelineMetric(org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric) HashMap(java.util.HashMap) TimelineEntity(org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity) HashSet(java.util.HashSet)

Example 7 with TimelineMetric

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric in project hadoop by apache.

the class TestFlowDataGenerator method getEntityMetricsApp1Complete.

static TimelineEntity getEntityMetricsApp1Complete(long insertTs, Configuration c1) {
    TimelineEntity entity = new TimelineEntity();
    String id = "flowRunMetrics_test";
    String type = TimelineEntityType.YARN_APPLICATION.toString();
    entity.setId(id);
    entity.setType(type);
    long cTime = 1425016501000L;
    entity.setCreatedTime(cTime);
    // add metrics
    Set<TimelineMetric> metrics = new HashSet<>();
    TimelineMetric m1 = new TimelineMetric();
    m1.setId(METRIC_1);
    Map<Long, Number> metricValues = new HashMap<Long, Number>();
    long ts = insertTs;
    metricValues.put(ts - 80000, 40L);
    m1.setType(Type.TIME_SERIES);
    m1.setValues(metricValues);
    metrics.add(m1);
    TimelineMetric m2 = new TimelineMetric();
    m2.setId(METRIC_2);
    metricValues = new HashMap<Long, Number>();
    ts = insertTs;
    metricValues.put(ts - 80000, 57L);
    m2.setType(Type.TIME_SERIES);
    m2.setValues(metricValues);
    metrics.add(m2);
    entity.addMetrics(metrics);
    TimelineEvent event = new TimelineEvent();
    event.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
    event.setTimestamp(insertTs);
    event.addInfo("done", "insertTs=" + insertTs);
    entity.addEvent(event);
    return entity;
}
Also used : TimelineEvent(org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent) TimelineMetric(org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric) HashMap(java.util.HashMap) TimelineEntity(org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity) HashSet(java.util.HashSet)

Example 8 with TimelineMetric

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric in project hadoop by apache.

the class TestFlowDataGenerator method getAFullEntity.

static TimelineEntity getAFullEntity(long ts, long endTs) {
    TimelineEntity entity = new TimelineEntity();
    String id = "flowRunFullEntity";
    String type = TimelineEntityType.YARN_APPLICATION.toString();
    entity.setId(id);
    entity.setType(type);
    entity.setCreatedTime(ts);
    // add metrics
    Set<TimelineMetric> metrics = new HashSet<>();
    TimelineMetric m1 = new TimelineMetric();
    m1.setId(METRIC_1);
    Map<Long, Number> metricValues = new HashMap<Long, Number>();
    metricValues.put(ts - 120000, 100000000L);
    metricValues.put(ts - 100000, 200000000L);
    metricValues.put(ts - 80000, 300000000L);
    metricValues.put(ts - 60000, 400000000L);
    metricValues.put(ts - 40000, 50000000000L);
    metricValues.put(ts - 20000, 60000000000L);
    m1.setType(Type.TIME_SERIES);
    m1.setValues(metricValues);
    metrics.add(m1);
    TimelineMetric m2 = new TimelineMetric();
    m2.setId(METRIC_2);
    metricValues = new HashMap<Long, Number>();
    metricValues.put(ts - 900000, 31L);
    metricValues.put(ts - 30000, 57L);
    m2.setType(Type.TIME_SERIES);
    m2.setValues(metricValues);
    metrics.add(m2);
    entity.addMetrics(metrics);
    TimelineEvent event = new TimelineEvent();
    event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
    event.setTimestamp(ts);
    String expKey = "foo_event";
    Object expVal = "test";
    event.addInfo(expKey, expVal);
    entity.addEvent(event);
    event = new TimelineEvent();
    event.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
    // start time + 6hrs
    long expTs = ts + 21600000;
    event.setTimestamp(expTs);
    event.addInfo(expKey, expVal);
    entity.addEvent(event);
    return entity;
}
Also used : TimelineEvent(org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent) TimelineMetric(org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric) HashMap(java.util.HashMap) TimelineEntity(org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity) HashSet(java.util.HashSet)

Example 9 with TimelineMetric

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric in project hadoop by apache.

the class TestHBaseTimelineStorageApps method testReadAppsMetricPrefix.

@Test
public void testReadAppsMetricPrefix() throws Exception {
    TimelineFilterList list = new TimelineFilterList(Operator.OR, new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
    TimelineEntity e1 = reader.getEntity(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1111111111_2222", TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineDataToRetrieve(null, list, null, null));
    assertNotNull(e1);
    assertEquals(1, e1.getMetrics().size());
    Set<TimelineEntity> es1 = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(), new TimelineDataToRetrieve(null, list, null, null));
    int metricCnt = 0;
    for (TimelineEntity entity : es1) {
        metricCnt += entity.getMetrics().size();
        for (TimelineMetric metric : entity.getMetrics()) {
            assertTrue("Metric Id returned should start with MAP1_", metric.getId().startsWith("MAP1_"));
        }
    }
    assertEquals(2, metricCnt);
}
Also used : TimelineMetric(org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric) TimelineFilterList(org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList) TimelineReaderContext(org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext) TimelineEntityFilters(org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters) TimelinePrefixFilter(org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter) TimelineEntity(org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity) TimelineDataToRetrieve(org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve) Test(org.junit.Test)

Example 10 with TimelineMetric

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric in project hadoop by apache.

the class TestHBaseTimelineStorageApps method testReadAppsMetricFilterPrefix.

@Test
public void testReadAppsMetricFilterPrefix() throws Exception {
    TimelineFilterList list = new TimelineFilterList(Operator.OR, new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
    TimelineFilterList metricFilterList = new TimelineFilterList();
    metricFilterList.addFilter(new TimelineCompareFilter(TimelineCompareOp.GREATER_OR_EQUAL, "MAP1_SLOT_MILLIS", 0L));
    Set<TimelineEntity> entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, null, metricFilterList, null), new TimelineDataToRetrieve(null, list, null, null));
    int metricCnt = 0;
    assertEquals(1, entities.size());
    for (TimelineEntity entity : entities) {
        metricCnt += entity.getMetrics().size();
    }
    assertEquals(1, metricCnt);
    TimelineFilterList list1 = new TimelineFilterList();
    list1.addFilter(new TimelineCompareFilter(TimelineCompareOp.GREATER_OR_EQUAL, "MAP1_SLOT_MILLIS", 50000000900L));
    TimelineFilterList list2 = new TimelineFilterList();
    list2.addFilter(new TimelineCompareFilter(TimelineCompareOp.LESS_THAN, "MAP_SLOT_MILLIS", 80000000000L));
    list2.addFilter(new TimelineCompareFilter(TimelineCompareOp.EQUAL, "MAP1_BYTES", 50));
    TimelineFilterList metricsToRetrieve = new TimelineFilterList(Operator.OR, new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
    TimelineFilterList metricFilterList1 = new TimelineFilterList(Operator.OR, list1, list2);
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, null, metricFilterList1, null), new TimelineDataToRetrieve(null, metricsToRetrieve, null, null));
    metricCnt = 0;
    assertEquals(2, entities.size());
    for (TimelineEntity entity : entities) {
        metricCnt += entity.getMetrics().size();
        for (TimelineMetric metric : entity.getMetrics()) {
            assertTrue("Metric Id returned should start with MAP1_", metric.getId().startsWith("MAP1_"));
        }
    }
    assertEquals(2, metricCnt);
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, null, metricFilterList1, null), new TimelineDataToRetrieve(null, metricsToRetrieve, EnumSet.of(Field.METRICS), Integer.MAX_VALUE));
    metricCnt = 0;
    int metricValCnt = 0;
    assertEquals(2, entities.size());
    for (TimelineEntity entity : entities) {
        metricCnt += entity.getMetrics().size();
        for (TimelineMetric metric : entity.getMetrics()) {
            metricValCnt += metric.getValues().size();
            assertTrue("Metric Id returned should start with MAP1_", metric.getId().startsWith("MAP1_"));
        }
    }
    assertEquals(2, metricCnt);
    assertEquals(7, metricValCnt);
}
Also used : TimelineMetric(org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric) TimelineFilterList(org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList) TimelineReaderContext(org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext) TimelineEntityFilters(org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters) TimelineCompareFilter(org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareFilter) TimelinePrefixFilter(org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter) TimelineEntity(org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity) TimelineDataToRetrieve(org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve) Test(org.junit.Test)

Aggregations

TimelineMetric (org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric)47 TimelineEntity (org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity)33 Test (org.junit.Test)22 HashSet (java.util.HashSet)21 HashMap (java.util.HashMap)17 TimelineEntities (org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities)13 TimelineEvent (org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent)12 Set (java.util.Set)11 Client (com.sun.jersey.api.client.Client)9 ClientResponse (com.sun.jersey.api.client.ClientResponse)9 URI (java.net.URI)9 Configuration (org.apache.hadoop.conf.Configuration)9 TimelineDataToRetrieve (org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve)9 TimelineReaderContext (org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext)9 TimelineEntityFilters (org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters)7 GenericType (com.sun.jersey.api.client.GenericType)5 Map (java.util.Map)5 TimelineFilterList (org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList)5 TimelinePrefixFilter (org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter)5 IOException (java.io.IOException)4