Examples with TimelineEntity org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity used on opensource projects

Example 81 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity in project hadoop by apache.

the class TestHBaseTimelineStorageApps method testReadAppsRelatesTo.

@Test
public void testReadAppsRelatesTo() throws Exception {
    TimelineFilterList rt = new TimelineFilterList(Operator.OR);
    rt.addFilter(new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL, "container2", new HashSet<Object>(Arrays.asList("relatesto7"))));
    rt.addFilter(new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL, "container1", new HashSet<Object>(Arrays.asList("relatesto4"))));
    Set<TimelineEntity> entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, rt, null, null, null, null, null), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
    assertEquals(2, entities.size());
    int relatesToCnt = 0;
    for (TimelineEntity timelineEntity : entities) {
        relatesToCnt += timelineEntity.getRelatesToEntities().size();
        if (!timelineEntity.getId().equals("application_1111111111_2222") && !timelineEntity.getId().equals("application_1111111111_4444")) {
            Assert.fail("Entity ids' should have been application_1111111111_2222" + " and application_1111111111_4444");
        }
    }
    assertEquals(3, relatesToCnt);
    TimelineFilterList rt1 = new TimelineFilterList();
    rt1.addFilter(new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL, "container", new HashSet<Object>(Arrays.asList("relatesto1"))));
    rt1.addFilter(new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL, "container", new HashSet<Object>(Arrays.asList("relatesto3"))));
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, rt1, null, null, null, null, null), new TimelineDataToRetrieve());
    assertEquals(1, entities.size());
    relatesToCnt = 0;
    for (TimelineEntity timelineEntity : entities) {
        relatesToCnt += timelineEntity.getRelatesToEntities().size();
        if (!timelineEntity.getId().equals("application_1111111111_3333")) {
            Assert.fail("Entity id should have been application_1111111111_3333");
        }
    }
    assertEquals(0, relatesToCnt);
    TimelineFilterList rt2 = new TimelineFilterList(Operator.OR);
    rt2.addFilter(new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL, "container2", new HashSet<Object>(Arrays.asList("relatesto7"))));
    rt2.addFilter(new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL, "container1", new HashSet<Object>(Arrays.asList("relatesto4"))));
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, rt2, null, null, null, null, null), new TimelineDataToRetrieve());
    assertEquals(2, entities.size());
    relatesToCnt = 0;
    for (TimelineEntity timelineEntity : entities) {
        relatesToCnt += timelineEntity.getRelatesToEntities().size();
        if (!timelineEntity.getId().equals("application_1111111111_2222") && !timelineEntity.getId().equals("application_1111111111_4444")) {
            Assert.fail("Entity ids' should have been application_1111111111_2222" + " and application_1111111111_4444");
        }
    }
    assertEquals(0, relatesToCnt);
    TimelineFilterList rt3 = new TimelineFilterList();
    rt3.addFilter(new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL, "container", new HashSet<Object>(Arrays.asList("relatesto1", "relatesto3"))));
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, rt3, null, null, null, null, null), new TimelineDataToRetrieve());
    assertEquals(1, entities.size());
    relatesToCnt = 0;
    for (TimelineEntity timelineEntity : entities) {
        relatesToCnt += timelineEntity.getRelatesToEntities().size();
        if (!timelineEntity.getId().equals("application_1111111111_2222")) {
            Assert.fail("Entity id should have been application_1111111111_2222");
        }
    }
    assertEquals(0, relatesToCnt);
    TimelineFilterList rt4 = new TimelineFilterList();
    rt4.addFilter(new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL, "container", new HashSet<Object>(Arrays.asList("relatesto1"))));
    rt4.addFilter(new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL, "dummy_container", new HashSet<Object>(Arrays.asList("relatesto5"))));
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, rt4, null, null, null, null, null), new TimelineDataToRetrieve());
    assertEquals(0, entities.size());
    TimelineFilterList rt5 = new TimelineFilterList();
    rt5.addFilter(new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL, "container", new HashSet<Object>(Arrays.asList("relatedto1", "relatesto8"))));
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, rt5, null, null, null, null, null), new TimelineDataToRetrieve());
    assertEquals(0, entities.size());
    TimelineFilterList list1 = new TimelineFilterList();
    list1.addFilter(new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL, "container2", new HashSet<Object>(Arrays.asList("relatesto7"))));
    list1.addFilter(new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL, "dummy_container", new HashSet<Object>(Arrays.asList("relatesto4"))));
    TimelineFilterList list2 = new TimelineFilterList();
    list2.addFilter(new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL, "container1", new HashSet<Object>(Arrays.asList("relatesto4"))));
    TimelineFilterList rt6 = new TimelineFilterList(Operator.OR, list1, list2);
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, rt6, null, null, null, null, null), new TimelineDataToRetrieve());
    assertEquals(1, entities.size());
    relatesToCnt = 0;
    for (TimelineEntity timelineEntity : entities) {
        relatesToCnt += timelineEntity.getRelatesToEntities().size();
        if (!timelineEntity.getId().equals("application_1111111111_2222")) {
            Assert.fail("Entity id should have been application_1111111111_2222");
        }
    }
    assertEquals(0, relatesToCnt);
    TimelineFilterList list3 = new TimelineFilterList();
    list3.addFilter(new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL, "container", new HashSet<Object>(Arrays.asList("relatesto1"))));
    list3.addFilter(new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL, "container1", new HashSet<Object>(Arrays.asList("relatesto4"))));
    TimelineFilterList list4 = new TimelineFilterList();
    list4.addFilter(new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL, "container", new HashSet<Object>(Arrays.asList("relatesto1"))));
    list4.addFilter(new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL, "container", new HashSet<Object>(Arrays.asList("relatesto2"))));
    TimelineFilterList combinedList = new TimelineFilterList(Operator.OR, list3, list4);
    TimelineFilterList rt7 = new TimelineFilterList(Operator.AND, combinedList, new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL, "container", new HashSet<Object>(Arrays.asList("relatesto3"))));
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, rt7, null, null, null, null, null), new TimelineDataToRetrieve());
    assertEquals(1, entities.size());
    relatesToCnt = 0;
    for (TimelineEntity timelineEntity : entities) {
        relatesToCnt += timelineEntity.getRelatesToEntities().size();
        if (!timelineEntity.getId().equals("application_1111111111_3333")) {
            Assert.fail("Entity id should have been application_1111111111_3333");
        }
    }
    assertEquals(0, relatesToCnt);
}

Example 82 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity in project hadoop by apache.

the class TestHBaseTimelineStorageApps method testReadApps.

@Test
public void testReadApps() throws Exception {
    TimelineEntity entity = reader.getEntity(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1111111111_2222", TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
    assertNotNull(entity);
    assertEquals(3, entity.getConfigs().size());
    assertEquals(1, entity.getIsRelatedToEntities().size());
    Set<TimelineEntity> entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
    assertEquals(3, entities.size());
    int cfgCnt = 0;
    int metricCnt = 0;
    int infoCnt = 0;
    int eventCnt = 0;
    int relatesToCnt = 0;
    int isRelatedToCnt = 0;
    for (TimelineEntity timelineEntity : entities) {
        cfgCnt += (timelineEntity.getConfigs() == null) ? 0 : timelineEntity.getConfigs().size();
        metricCnt += (timelineEntity.getMetrics() == null) ? 0 : timelineEntity.getMetrics().size();
        infoCnt += (timelineEntity.getInfo() == null) ? 0 : timelineEntity.getInfo().size();
        eventCnt += (timelineEntity.getEvents() == null) ? 0 : timelineEntity.getEvents().size();
        relatesToCnt += (timelineEntity.getRelatesToEntities() == null) ? 0 : timelineEntity.getRelatesToEntities().size();
        isRelatedToCnt += (timelineEntity.getIsRelatedToEntities() == null) ? 0 : timelineEntity.getIsRelatedToEntities().size();
    }
    assertEquals(5, cfgCnt);
    assertEquals(3, metricCnt);
    assertEquals(5, infoCnt);
    assertEquals(4, eventCnt);
    assertEquals(4, relatesToCnt);
    assertEquals(4, isRelatedToCnt);
}

Example 83 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity in project hadoop by apache.

the class TestFlowDataGenerator method getEntityMetricsApp1.

static TimelineEntity getEntityMetricsApp1(long insertTs) {
    TimelineEntity entity = new TimelineEntity();
    String id = "flowRunMetrics_test";
    String type = TimelineEntityType.YARN_APPLICATION.toString();
    entity.setId(id);
    entity.setType(type);
    long cTime = 1425016501000L;
    entity.setCreatedTime(cTime);
    // add metrics
    Set<TimelineMetric> metrics = new HashSet<>();
    TimelineMetric m1 = new TimelineMetric();
    m1.setId(METRIC_1);
    Map<Long, Number> metricValues = new HashMap<Long, Number>();
    long ts = insertTs;
    metricValues.put(ts - 100000, 2L);
    metricValues.put(ts - 80000, 40L);
    m1.setType(Type.TIME_SERIES);
    m1.setValues(metricValues);
    metrics.add(m1);
    TimelineMetric m2 = new TimelineMetric();
    m2.setId(METRIC_2);
    metricValues = new HashMap<Long, Number>();
    ts = insertTs;
    metricValues.put(ts - 100000, 31L);
    metricValues.put(ts - 80000, 57L);
    m2.setType(Type.TIME_SERIES);
    m2.setValues(metricValues);
    metrics.add(m2);
    entity.addMetrics(metrics);
    TimelineEvent event = new TimelineEvent();
    event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
    long endTs = 1439379885000L;
    event.setTimestamp(endTs);
    String expKey = "foo_event_greater";
    String expVal = "test_app_greater";
    event.addInfo(expKey, expVal);
    entity.addEvent(event);
    return entity;
}

Example 84 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity in project hadoop by apache.

the class TestHBaseStorageFlowActivity method testWriteFlowRunMinMax.

/**
   * Writes 4 timeline entities belonging to one flow run through the
   * {@link HBaseTimelineWriterImpl}
   *
   * Checks the flow run table contents
   *
   * The first entity has a created event, metrics and a finish event.
   *
   * The second entity has a created event and this is the entity with smallest
   * start time. This should be the start time for the flow run.
   *
   * The third entity has a finish event and this is the entity with the max end
   * time. This should be the end time for the flow run.
   *
   * The fourth entity has a created event which has a start time that is
   * greater than min start time.
   *
   * The test also checks in the flow activity table that one entry has been
   * made for all of these 4 application entities since they belong to the same
   * flow run.
   */
@Test
public void testWriteFlowRunMinMax() throws Exception {
    TimelineEntities te = new TimelineEntities();
    te.addEntity(TestFlowDataGenerator.getEntity1());
    HBaseTimelineWriterImpl hbi = null;
    Configuration c1 = util.getConfiguration();
    String cluster = "testWriteFlowRunMinMaxToHBase_cluster1";
    String user = "testWriteFlowRunMinMaxToHBase_user1";
    String flow = "testing_flowRun_flow_name";
    String flowVersion = "CF7022C10F1354";
    long runid = 1002345678919L;
    String appName = "application_100000000000_1111";
    long minStartTs = 1424995200300L;
    long greaterStartTs = 1424995200300L + 864000L;
    long endTs = 1424995200300L + 86000000L;
    TimelineEntity entityMinStartTime = TestFlowDataGenerator.getEntityMinStartTime(minStartTs);
    try {
        hbi = new HBaseTimelineWriterImpl();
        hbi.init(c1);
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // write another entity with the right min start time
        te = new TimelineEntities();
        te.addEntity(entityMinStartTime);
        appName = "application_100000000000_3333";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // writer another entity for max end time
        TimelineEntity entityMaxEndTime = TestFlowDataGenerator.getEntityMaxEndTime(endTs);
        te = new TimelineEntities();
        te.addEntity(entityMaxEndTime);
        appName = "application_100000000000_4444";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // writer another entity with greater start time
        TimelineEntity entityGreaterStartTime = TestFlowDataGenerator.getEntityGreaterStartTime(greaterStartTs);
        te = new TimelineEntities();
        te.addEntity(entityGreaterStartTime);
        appName = "application_1000000000000000_2222";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // flush everything to hbase
        hbi.flush();
    } finally {
        if (hbi != null) {
            hbi.close();
        }
    }
    Connection conn = ConnectionFactory.createConnection(c1);
    // check in flow activity table
    Table table1 = conn.getTable(TableName.valueOf(FlowActivityTable.DEFAULT_TABLE_NAME));
    byte[] startRow = new FlowActivityRowKey(cluster, minStartTs, user, flow).getRowKey();
    Get g = new Get(startRow);
    Result r1 = table1.get(g);
    assertNotNull(r1);
    assertTrue(!r1.isEmpty());
    Map<byte[], byte[]> values = r1.getFamilyMap(FlowActivityColumnFamily.INFO.getBytes());
    assertEquals(1, values.size());
    byte[] row = r1.getRow();
    FlowActivityRowKey flowActivityRowKey = FlowActivityRowKey.parseRowKey(row);
    assertNotNull(flowActivityRowKey);
    assertEquals(cluster, flowActivityRowKey.getClusterId());
    assertEquals(user, flowActivityRowKey.getUserId());
    assertEquals(flow, flowActivityRowKey.getFlowName());
    Long dayTs = HBaseTimelineStorageUtils.getTopOfTheDayTimestamp(minStartTs);
    assertEquals(dayTs, flowActivityRowKey.getDayTimestamp());
    assertEquals(1, values.size());
    checkFlowActivityRunId(runid, flowVersion, values);
    // use the timeline reader to verify data
    HBaseTimelineReaderImpl hbr = null;
    try {
        hbr = new HBaseTimelineReaderImpl();
        hbr.init(c1);
        hbr.start();
        // get the flow activity entity
        Set<TimelineEntity> entities = hbr.getEntities(new TimelineReaderContext(cluster, null, null, null, null, TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null), new TimelineEntityFilters(10L, null, null, null, null, null, null, null, null), new TimelineDataToRetrieve());
        assertEquals(1, entities.size());
        for (TimelineEntity e : entities) {
            FlowActivityEntity flowActivity = (FlowActivityEntity) e;
            assertEquals(cluster, flowActivity.getCluster());
            assertEquals(user, flowActivity.getUser());
            assertEquals(flow, flowActivity.getFlowName());
            assertEquals(dayTs, Long.valueOf(flowActivity.getDate().getTime()));
            Set<FlowRunEntity> flowRuns = flowActivity.getFlowRuns();
            assertEquals(1, flowRuns.size());
        }
    } finally {
        if (hbr != null) {
            hbr.close();
        }
    }
}

Example 85 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity in project hadoop by apache.

the class TestHBaseStorageFlowRun method testWriteFlowRunMetricsOneFlow.

/**
   * Writes two application entities of the same flow run. Each application has
   * two metrics: slot millis and hdfs bytes read. Each metric has values at two
   * timestamps.
   *
   * Checks the metric values of the flow in the flow run table. Flow metric
   * values should be the sum of individual metric values that belong to the
   * latest timestamp for that metric
   */
@Test
public void testWriteFlowRunMetricsOneFlow() throws Exception {
    String cluster = "testWriteFlowRunMetricsOneFlow_cluster1";
    String user = "testWriteFlowRunMetricsOneFlow_user1";
    String flow = "testing_flowRun_metrics_flow_name";
    String flowVersion = "CF7022C10F1354";
    long runid = 1002345678919L;
    TimelineEntities te = new TimelineEntities();
    TimelineEntity entityApp1 = TestFlowDataGenerator.getEntityMetricsApp1(System.currentTimeMillis());
    te.addEntity(entityApp1);
    HBaseTimelineWriterImpl hbi = null;
    Configuration c1 = util.getConfiguration();
    try {
        hbi = new HBaseTimelineWriterImpl();
        hbi.init(c1);
        String appName = "application_11111111111111_1111";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // write another application with same metric to this flow
        te = new TimelineEntities();
        TimelineEntity entityApp2 = TestFlowDataGenerator.getEntityMetricsApp2(System.currentTimeMillis());
        te.addEntity(entityApp2);
        appName = "application_11111111111111_2222";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        hbi.flush();
    } finally {
        if (hbi != null) {
            hbi.close();
        }
    }
    // check flow run
    checkFlowRunTable(cluster, user, flow, runid, c1);
    // check various batch limits in scanning the table for this flow
    checkFlowRunTableBatchLimit(cluster, user, flow, runid, c1);
    // use the timeline reader to verify data
    HBaseTimelineReaderImpl hbr = null;
    try {
        hbr = new HBaseTimelineReaderImpl();
        hbr.init(c1);
        hbr.start();
        TimelineEntity entity = hbr.getEntity(new TimelineReaderContext(cluster, user, flow, runid, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), new TimelineDataToRetrieve());
        assertTrue(TimelineEntityType.YARN_FLOW_RUN.matches(entity.getType()));
        Set<TimelineMetric> metrics = entity.getMetrics();
        assertEquals(2, metrics.size());
        for (TimelineMetric metric : metrics) {
            String id = metric.getId();
            Map<Long, Number> values = metric.getValues();
            assertEquals(1, values.size());
            Number value = null;
            for (Number n : values.values()) {
                value = n;
            }
            switch(id) {
                case METRIC1:
                    assertEquals(141L, value);
                    break;
                case METRIC2:
                    assertEquals(57L, value);
                    break;
                default:
                    fail("unrecognized metric: " + id);
            }
        }
    } finally {
        if (hbr != null) {
            hbr.close();
        }
    }
}