Examples with TimelineDataToRetrieve org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve used on opensource projects

Example 36 with TimelineDataToRetrieve

use of org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve in project hadoop by apache.

the class TestHBaseTimelineStorageApps method testReadApps.

@Test
public void testReadApps() throws Exception {
    TimelineEntity entity = reader.getEntity(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1111111111_2222", TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
    assertNotNull(entity);
    assertEquals(3, entity.getConfigs().size());
    assertEquals(1, entity.getIsRelatedToEntities().size());
    Set<TimelineEntity> entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
    assertEquals(3, entities.size());
    int cfgCnt = 0;
    int metricCnt = 0;
    int infoCnt = 0;
    int eventCnt = 0;
    int relatesToCnt = 0;
    int isRelatedToCnt = 0;
    for (TimelineEntity timelineEntity : entities) {
        cfgCnt += (timelineEntity.getConfigs() == null) ? 0 : timelineEntity.getConfigs().size();
        metricCnt += (timelineEntity.getMetrics() == null) ? 0 : timelineEntity.getMetrics().size();
        infoCnt += (timelineEntity.getInfo() == null) ? 0 : timelineEntity.getInfo().size();
        eventCnt += (timelineEntity.getEvents() == null) ? 0 : timelineEntity.getEvents().size();
        relatesToCnt += (timelineEntity.getRelatesToEntities() == null) ? 0 : timelineEntity.getRelatesToEntities().size();
        isRelatedToCnt += (timelineEntity.getIsRelatedToEntities() == null) ? 0 : timelineEntity.getIsRelatedToEntities().size();
    }
    assertEquals(5, cfgCnt);
    assertEquals(3, metricCnt);
    assertEquals(5, infoCnt);
    assertEquals(4, eventCnt);
    assertEquals(4, relatesToCnt);
    assertEquals(4, isRelatedToCnt);
}

Example 37 with TimelineDataToRetrieve

use of org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve in project hadoop by apache.

the class TestHBaseStorageFlowActivity method testWriteFlowRunMinMax.

/**
   * Writes 4 timeline entities belonging to one flow run through the
   * {@link HBaseTimelineWriterImpl}
   *
   * Checks the flow run table contents
   *
   * The first entity has a created event, metrics and a finish event.
   *
   * The second entity has a created event and this is the entity with smallest
   * start time. This should be the start time for the flow run.
   *
   * The third entity has a finish event and this is the entity with the max end
   * time. This should be the end time for the flow run.
   *
   * The fourth entity has a created event which has a start time that is
   * greater than min start time.
   *
   * The test also checks in the flow activity table that one entry has been
   * made for all of these 4 application entities since they belong to the same
   * flow run.
   */
@Test
public void testWriteFlowRunMinMax() throws Exception {
    TimelineEntities te = new TimelineEntities();
    te.addEntity(TestFlowDataGenerator.getEntity1());
    HBaseTimelineWriterImpl hbi = null;
    Configuration c1 = util.getConfiguration();
    String cluster = "testWriteFlowRunMinMaxToHBase_cluster1";
    String user = "testWriteFlowRunMinMaxToHBase_user1";
    String flow = "testing_flowRun_flow_name";
    String flowVersion = "CF7022C10F1354";
    long runid = 1002345678919L;
    String appName = "application_100000000000_1111";
    long minStartTs = 1424995200300L;
    long greaterStartTs = 1424995200300L + 864000L;
    long endTs = 1424995200300L + 86000000L;
    TimelineEntity entityMinStartTime = TestFlowDataGenerator.getEntityMinStartTime(minStartTs);
    try {
        hbi = new HBaseTimelineWriterImpl();
        hbi.init(c1);
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // write another entity with the right min start time
        te = new TimelineEntities();
        te.addEntity(entityMinStartTime);
        appName = "application_100000000000_3333";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // writer another entity for max end time
        TimelineEntity entityMaxEndTime = TestFlowDataGenerator.getEntityMaxEndTime(endTs);
        te = new TimelineEntities();
        te.addEntity(entityMaxEndTime);
        appName = "application_100000000000_4444";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // writer another entity with greater start time
        TimelineEntity entityGreaterStartTime = TestFlowDataGenerator.getEntityGreaterStartTime(greaterStartTs);
        te = new TimelineEntities();
        te.addEntity(entityGreaterStartTime);
        appName = "application_1000000000000000_2222";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // flush everything to hbase
        hbi.flush();
    } finally {
        if (hbi != null) {
            hbi.close();
        }
    }
    Connection conn = ConnectionFactory.createConnection(c1);
    // check in flow activity table
    Table table1 = conn.getTable(TableName.valueOf(FlowActivityTable.DEFAULT_TABLE_NAME));
    byte[] startRow = new FlowActivityRowKey(cluster, minStartTs, user, flow).getRowKey();
    Get g = new Get(startRow);
    Result r1 = table1.get(g);
    assertNotNull(r1);
    assertTrue(!r1.isEmpty());
    Map<byte[], byte[]> values = r1.getFamilyMap(FlowActivityColumnFamily.INFO.getBytes());
    assertEquals(1, values.size());
    byte[] row = r1.getRow();
    FlowActivityRowKey flowActivityRowKey = FlowActivityRowKey.parseRowKey(row);
    assertNotNull(flowActivityRowKey);
    assertEquals(cluster, flowActivityRowKey.getClusterId());
    assertEquals(user, flowActivityRowKey.getUserId());
    assertEquals(flow, flowActivityRowKey.getFlowName());
    Long dayTs = HBaseTimelineStorageUtils.getTopOfTheDayTimestamp(minStartTs);
    assertEquals(dayTs, flowActivityRowKey.getDayTimestamp());
    assertEquals(1, values.size());
    checkFlowActivityRunId(runid, flowVersion, values);
    // use the timeline reader to verify data
    HBaseTimelineReaderImpl hbr = null;
    try {
        hbr = new HBaseTimelineReaderImpl();
        hbr.init(c1);
        hbr.start();
        // get the flow activity entity
        Set<TimelineEntity> entities = hbr.getEntities(new TimelineReaderContext(cluster, null, null, null, null, TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null), new TimelineEntityFilters(10L, null, null, null, null, null, null, null, null), new TimelineDataToRetrieve());
        assertEquals(1, entities.size());
        for (TimelineEntity e : entities) {
            FlowActivityEntity flowActivity = (FlowActivityEntity) e;
            assertEquals(cluster, flowActivity.getCluster());
            assertEquals(user, flowActivity.getUser());
            assertEquals(flow, flowActivity.getFlowName());
            assertEquals(dayTs, Long.valueOf(flowActivity.getDate().getTime()));
            Set<FlowRunEntity> flowRuns = flowActivity.getFlowRuns();
            assertEquals(1, flowRuns.size());
        }
    } finally {
        if (hbr != null) {
            hbr.close();
        }
    }
}

Example 38 with TimelineDataToRetrieve

use of org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve in project hadoop by apache.

the class TestHBaseStorageFlowRun method testWriteFlowRunMetricsOneFlow.

/**
   * Writes two application entities of the same flow run. Each application has
   * two metrics: slot millis and hdfs bytes read. Each metric has values at two
   * timestamps.
   *
   * Checks the metric values of the flow in the flow run table. Flow metric
   * values should be the sum of individual metric values that belong to the
   * latest timestamp for that metric
   */
@Test
public void testWriteFlowRunMetricsOneFlow() throws Exception {
    String cluster = "testWriteFlowRunMetricsOneFlow_cluster1";
    String user = "testWriteFlowRunMetricsOneFlow_user1";
    String flow = "testing_flowRun_metrics_flow_name";
    String flowVersion = "CF7022C10F1354";
    long runid = 1002345678919L;
    TimelineEntities te = new TimelineEntities();
    TimelineEntity entityApp1 = TestFlowDataGenerator.getEntityMetricsApp1(System.currentTimeMillis());
    te.addEntity(entityApp1);
    HBaseTimelineWriterImpl hbi = null;
    Configuration c1 = util.getConfiguration();
    try {
        hbi = new HBaseTimelineWriterImpl();
        hbi.init(c1);
        String appName = "application_11111111111111_1111";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // write another application with same metric to this flow
        te = new TimelineEntities();
        TimelineEntity entityApp2 = TestFlowDataGenerator.getEntityMetricsApp2(System.currentTimeMillis());
        te.addEntity(entityApp2);
        appName = "application_11111111111111_2222";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        hbi.flush();
    } finally {
        if (hbi != null) {
            hbi.close();
        }
    }
    // check flow run
    checkFlowRunTable(cluster, user, flow, runid, c1);
    // check various batch limits in scanning the table for this flow
    checkFlowRunTableBatchLimit(cluster, user, flow, runid, c1);
    // use the timeline reader to verify data
    HBaseTimelineReaderImpl hbr = null;
    try {
        hbr = new HBaseTimelineReaderImpl();
        hbr.init(c1);
        hbr.start();
        TimelineEntity entity = hbr.getEntity(new TimelineReaderContext(cluster, user, flow, runid, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), new TimelineDataToRetrieve());
        assertTrue(TimelineEntityType.YARN_FLOW_RUN.matches(entity.getType()));
        Set<TimelineMetric> metrics = entity.getMetrics();
        assertEquals(2, metrics.size());
        for (TimelineMetric metric : metrics) {
            String id = metric.getId();
            Map<Long, Number> values = metric.getValues();
            assertEquals(1, values.size());
            Number value = null;
            for (Number n : values.values()) {
                value = n;
            }
            switch(id) {
                case METRIC1:
                    assertEquals(141L, value);
                    break;
                case METRIC2:
                    assertEquals(57L, value);
                    break;
                default:
                    fail("unrecognized metric: " + id);
            }
        }
    } finally {
        if (hbr != null) {
            hbr.close();
        }
    }
}

Example 39 with TimelineDataToRetrieve

use of org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve in project hadoop by apache.

the class TestHBaseStorageFlowRun method testFilterFlowRunsByCreatedTime.

@Test
public void testFilterFlowRunsByCreatedTime() throws Exception {
    String cluster = "cluster2";
    String user = "user2";
    String flow = "flow_name2";
    TimelineEntities te = new TimelineEntities();
    TimelineEntity entityApp1 = TestFlowDataGenerator.getEntityMetricsApp1(System.currentTimeMillis());
    entityApp1.setCreatedTime(1425016501000L);
    te.addEntity(entityApp1);
    HBaseTimelineWriterImpl hbi = null;
    Configuration c1 = util.getConfiguration();
    try {
        hbi = new HBaseTimelineWriterImpl();
        hbi.init(c1);
        hbi.write(cluster, user, flow, "CF7022C10F1354", 1002345678919L, "application_11111111111111_1111", te);
        // write another application with same metric to this flow
        te = new TimelineEntities();
        TimelineEntity entityApp2 = TestFlowDataGenerator.getEntityMetricsApp2(System.currentTimeMillis());
        entityApp2.setCreatedTime(1425016502000L);
        te.addEntity(entityApp2);
        hbi.write(cluster, user, flow, "CF7022C10F1354", 1002345678918L, "application_11111111111111_2222", te);
        hbi.flush();
    } finally {
        if (hbi != null) {
            hbi.close();
        }
    }
    // use the timeline reader to verify data
    HBaseTimelineReaderImpl hbr = null;
    try {
        hbr = new HBaseTimelineReaderImpl();
        hbr.init(c1);
        hbr.start();
        Set<TimelineEntity> entities = hbr.getEntities(new TimelineReaderContext(cluster, user, flow, null, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), new TimelineEntityFilters(null, 1425016501000L, 1425016502001L, null, null, null, null, null, null), new TimelineDataToRetrieve());
        assertEquals(2, entities.size());
        for (TimelineEntity entity : entities) {
            if (!entity.getId().equals("user2@flow_name2/1002345678918") && !entity.getId().equals("user2@flow_name2/1002345678919")) {
                fail("Entities with flow runs 1002345678918 and 1002345678919" + "should be present.");
            }
        }
        entities = hbr.getEntities(new TimelineReaderContext(cluster, user, flow, null, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), new TimelineEntityFilters(null, 1425016501050L, null, null, null, null, null, null, null), new TimelineDataToRetrieve());
        assertEquals(1, entities.size());
        for (TimelineEntity entity : entities) {
            if (!entity.getId().equals("user2@flow_name2/1002345678918")) {
                fail("Entity with flow run 1002345678918 should be present.");
            }
        }
        entities = hbr.getEntities(new TimelineReaderContext(cluster, user, flow, null, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), new TimelineEntityFilters(null, null, 1425016501050L, null, null, null, null, null, null), new TimelineDataToRetrieve());
        assertEquals(1, entities.size());
        for (TimelineEntity entity : entities) {
            if (!entity.getId().equals("user2@flow_name2/1002345678919")) {
                fail("Entity with flow run 1002345678919 should be present.");
            }
        }
    } finally {
        if (hbr != null) {
            hbr.close();
        }
    }
}

Example 40 with TimelineDataToRetrieve

use of org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve in project hadoop by apache.

the class TestHBaseStorageFlowRun method testWriteFlowRunMinMax.

/**
   * Writes 4 timeline entities belonging to one flow run through the
   * {@link HBaseTimelineWriterImpl}
   *
   * Checks the flow run table contents
   *
   * The first entity has a created event, metrics and a finish event.
   *
   * The second entity has a created event and this is the entity with smallest
   * start time. This should be the start time for the flow run.
   *
   * The third entity has a finish event and this is the entity with the max end
   * time. This should be the end time for the flow run.
   *
   * The fourth entity has a created event which has a start time that is
   * greater than min start time.
   *
   */
@Test
public void testWriteFlowRunMinMax() throws Exception {
    TimelineEntities te = new TimelineEntities();
    te.addEntity(TestFlowDataGenerator.getEntity1());
    HBaseTimelineWriterImpl hbi = null;
    Configuration c1 = util.getConfiguration();
    String cluster = "testWriteFlowRunMinMaxToHBase_cluster1";
    String user = "testWriteFlowRunMinMaxToHBase_user1";
    String flow = "testing_flowRun_flow_name";
    String flowVersion = "CF7022C10F1354";
    long runid = 1002345678919L;
    String appName = "application_100000000000_1111";
    long minStartTs = 1425026900000L;
    long greaterStartTs = 30000000000000L;
    long endTs = 1439750690000L;
    TimelineEntity entityMinStartTime = TestFlowDataGenerator.getEntityMinStartTime(minStartTs);
    try {
        hbi = new HBaseTimelineWriterImpl();
        hbi.init(c1);
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // write another entity with the right min start time
        te = new TimelineEntities();
        te.addEntity(entityMinStartTime);
        appName = "application_100000000000_3333";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // writer another entity for max end time
        TimelineEntity entityMaxEndTime = TestFlowDataGenerator.getEntityMaxEndTime(endTs);
        te = new TimelineEntities();
        te.addEntity(entityMaxEndTime);
        appName = "application_100000000000_4444";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // writer another entity with greater start time
        TimelineEntity entityGreaterStartTime = TestFlowDataGenerator.getEntityGreaterStartTime(greaterStartTs);
        te = new TimelineEntities();
        te.addEntity(entityGreaterStartTime);
        appName = "application_1000000000000000_2222";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // flush everything to hbase
        hbi.flush();
    } finally {
        if (hbi != null) {
            hbi.close();
        }
    }
    Connection conn = ConnectionFactory.createConnection(c1);
    // check in flow run table
    Table table1 = conn.getTable(TableName.valueOf(FlowRunTable.DEFAULT_TABLE_NAME));
    // scan the table and see that we get back the right min and max
    // timestamps
    byte[] startRow = new FlowRunRowKey(cluster, user, flow, runid).getRowKey();
    Get g = new Get(startRow);
    g.addColumn(FlowRunColumnFamily.INFO.getBytes(), FlowRunColumn.MIN_START_TIME.getColumnQualifierBytes());
    g.addColumn(FlowRunColumnFamily.INFO.getBytes(), FlowRunColumn.MAX_END_TIME.getColumnQualifierBytes());
    Result r1 = table1.get(g);
    assertNotNull(r1);
    assertTrue(!r1.isEmpty());
    Map<byte[], byte[]> values = r1.getFamilyMap(FlowRunColumnFamily.INFO.getBytes());
    assertEquals(2, r1.size());
    long starttime = Bytes.toLong(values.get(FlowRunColumn.MIN_START_TIME.getColumnQualifierBytes()));
    assertEquals(minStartTs, starttime);
    assertEquals(endTs, Bytes.toLong(values.get(FlowRunColumn.MAX_END_TIME.getColumnQualifierBytes())));
    // use the timeline reader to verify data
    HBaseTimelineReaderImpl hbr = null;
    try {
        hbr = new HBaseTimelineReaderImpl();
        hbr.init(c1);
        hbr.start();
        // get the flow run entity
        TimelineEntity entity = hbr.getEntity(new TimelineReaderContext(cluster, user, flow, runid, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), new TimelineDataToRetrieve());
        assertTrue(TimelineEntityType.YARN_FLOW_RUN.matches(entity.getType()));
        FlowRunEntity flowRun = (FlowRunEntity) entity;
        assertEquals(minStartTs, flowRun.getStartTime());
        assertEquals(endTs, flowRun.getMaxEndTime());
    } finally {
        if (hbr != null) {
            hbr.close();
        }
    }
}