Examples with FlowRunEntity org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity used on opensource projects

Example 6 with FlowRunEntity

use of org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity in project hadoop by apache.

the class TimelineReaderManager method fillUID.

/**
   * Fill UID in the info field of entity based on the query(identified by
   * entity type).
   * @param entityType Entity type of query.
   * @param entity Timeline Entity.
   * @param context Context defining the query.
   */
private static void fillUID(TimelineEntityType entityType, TimelineEntity entity, TimelineReaderContext context) {
    if (entityType != null) {
        switch(entityType) {
            case YARN_FLOW_ACTIVITY:
                FlowActivityEntity activityEntity = (FlowActivityEntity) entity;
                context.setUserId(activityEntity.getUser());
                context.setFlowName(activityEntity.getFlowName());
                entity.setUID(UID_KEY, TimelineUIDConverter.FLOW_UID.encodeUID(context));
                return;
            case YARN_FLOW_RUN:
                FlowRunEntity runEntity = (FlowRunEntity) entity;
                context.setFlowRunId(runEntity.getRunId());
                entity.setUID(UID_KEY, TimelineUIDConverter.FLOWRUN_UID.encodeUID(context));
                return;
            case YARN_APPLICATION:
                context.setAppId(entity.getId());
                entity.setUID(UID_KEY, TimelineUIDConverter.APPLICATION_UID.encodeUID(context));
                return;
            default:
                break;
        }
    }
    context.setEntityType(entity.getType());
    context.setEntityId(entity.getId());
    entity.setUID(UID_KEY, TimelineUIDConverter.GENERIC_ENTITY_UID.encodeUID(context));
}

Example 7 with FlowRunEntity

use of org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity in project hadoop by apache.

the class FlowActivityEntityReader method parseEntity.

@Override
protected TimelineEntity parseEntity(Result result) throws IOException {
    FlowActivityRowKey rowKey = FlowActivityRowKey.parseRowKey(result.getRow());
    Long time = rowKey.getDayTimestamp();
    String user = rowKey.getUserId();
    String flowName = rowKey.getFlowName();
    FlowActivityEntity flowActivity = new FlowActivityEntity(getContext().getClusterId(), time, user, flowName);
    // set the id
    flowActivity.setId(flowActivity.getId());
    // get the list of run ids along with the version that are associated with
    // this flow on this day
    Map<Long, Object> runIdsMap = FlowActivityColumnPrefix.RUN_ID.readResults(result, longKeyConverter);
    for (Map.Entry<Long, Object> e : runIdsMap.entrySet()) {
        Long runId = e.getKey();
        String version = (String) e.getValue();
        FlowRunEntity flowRun = new FlowRunEntity();
        flowRun.setUser(user);
        flowRun.setName(flowName);
        flowRun.setRunId(runId);
        flowRun.setVersion(version);
        // set the id
        flowRun.setId(flowRun.getId());
        flowActivity.addFlowRun(flowRun);
    }
    return flowActivity;
}

Example 8 with FlowRunEntity

use of org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity in project hadoop by apache.

the class FlowRunEntityReader method parseEntity.

@Override
protected TimelineEntity parseEntity(Result result) throws IOException {
    TimelineReaderContext context = getContext();
    FlowRunEntity flowRun = new FlowRunEntity();
    flowRun.setUser(context.getUserId());
    flowRun.setName(context.getFlowName());
    if (isSingleEntityRead()) {
        flowRun.setRunId(context.getFlowRunId());
    } else {
        FlowRunRowKey rowKey = FlowRunRowKey.parseRowKey(result.getRow());
        flowRun.setRunId(rowKey.getFlowRunId());
    }
    // read the start time
    Long startTime = (Long) FlowRunColumn.MIN_START_TIME.readResult(result);
    if (startTime != null) {
        flowRun.setStartTime(startTime.longValue());
    }
    // read the end time if available
    Long endTime = (Long) FlowRunColumn.MAX_END_TIME.readResult(result);
    if (endTime != null) {
        flowRun.setMaxEndTime(endTime.longValue());
    }
    // read the flow version
    String version = (String) FlowRunColumn.FLOW_VERSION.readResult(result);
    if (version != null) {
        flowRun.setVersion(version);
    }
    // fieldsToRetrieve.
    if (isSingleEntityRead() || hasField(getDataToRetrieve().getFieldsToRetrieve(), Field.METRICS)) {
        readMetrics(flowRun, result, FlowRunColumnPrefix.METRIC);
    }
    // set the id
    flowRun.setId(flowRun.getId());
    return flowRun;
}

Example 9 with FlowRunEntity

use of org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity in project hadoop by apache.

the class TestHBaseStorageFlowActivity method testWriteFlowRunMinMax.

/**
   * Writes 4 timeline entities belonging to one flow run through the
   * {@link HBaseTimelineWriterImpl}
   *
   * Checks the flow run table contents
   *
   * The first entity has a created event, metrics and a finish event.
   *
   * The second entity has a created event and this is the entity with smallest
   * start time. This should be the start time for the flow run.
   *
   * The third entity has a finish event and this is the entity with the max end
   * time. This should be the end time for the flow run.
   *
   * The fourth entity has a created event which has a start time that is
   * greater than min start time.
   *
   * The test also checks in the flow activity table that one entry has been
   * made for all of these 4 application entities since they belong to the same
   * flow run.
   */
@Test
public void testWriteFlowRunMinMax() throws Exception {
    TimelineEntities te = new TimelineEntities();
    te.addEntity(TestFlowDataGenerator.getEntity1());
    HBaseTimelineWriterImpl hbi = null;
    Configuration c1 = util.getConfiguration();
    String cluster = "testWriteFlowRunMinMaxToHBase_cluster1";
    String user = "testWriteFlowRunMinMaxToHBase_user1";
    String flow = "testing_flowRun_flow_name";
    String flowVersion = "CF7022C10F1354";
    long runid = 1002345678919L;
    String appName = "application_100000000000_1111";
    long minStartTs = 1424995200300L;
    long greaterStartTs = 1424995200300L + 864000L;
    long endTs = 1424995200300L + 86000000L;
    TimelineEntity entityMinStartTime = TestFlowDataGenerator.getEntityMinStartTime(minStartTs);
    try {
        hbi = new HBaseTimelineWriterImpl();
        hbi.init(c1);
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // write another entity with the right min start time
        te = new TimelineEntities();
        te.addEntity(entityMinStartTime);
        appName = "application_100000000000_3333";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // writer another entity for max end time
        TimelineEntity entityMaxEndTime = TestFlowDataGenerator.getEntityMaxEndTime(endTs);
        te = new TimelineEntities();
        te.addEntity(entityMaxEndTime);
        appName = "application_100000000000_4444";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // writer another entity with greater start time
        TimelineEntity entityGreaterStartTime = TestFlowDataGenerator.getEntityGreaterStartTime(greaterStartTs);
        te = new TimelineEntities();
        te.addEntity(entityGreaterStartTime);
        appName = "application_1000000000000000_2222";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // flush everything to hbase
        hbi.flush();
    } finally {
        if (hbi != null) {
            hbi.close();
        }
    }
    Connection conn = ConnectionFactory.createConnection(c1);
    // check in flow activity table
    Table table1 = conn.getTable(TableName.valueOf(FlowActivityTable.DEFAULT_TABLE_NAME));
    byte[] startRow = new FlowActivityRowKey(cluster, minStartTs, user, flow).getRowKey();
    Get g = new Get(startRow);
    Result r1 = table1.get(g);
    assertNotNull(r1);
    assertTrue(!r1.isEmpty());
    Map<byte[], byte[]> values = r1.getFamilyMap(FlowActivityColumnFamily.INFO.getBytes());
    assertEquals(1, values.size());
    byte[] row = r1.getRow();
    FlowActivityRowKey flowActivityRowKey = FlowActivityRowKey.parseRowKey(row);
    assertNotNull(flowActivityRowKey);
    assertEquals(cluster, flowActivityRowKey.getClusterId());
    assertEquals(user, flowActivityRowKey.getUserId());
    assertEquals(flow, flowActivityRowKey.getFlowName());
    Long dayTs = HBaseTimelineStorageUtils.getTopOfTheDayTimestamp(minStartTs);
    assertEquals(dayTs, flowActivityRowKey.getDayTimestamp());
    assertEquals(1, values.size());
    checkFlowActivityRunId(runid, flowVersion, values);
    // use the timeline reader to verify data
    HBaseTimelineReaderImpl hbr = null;
    try {
        hbr = new HBaseTimelineReaderImpl();
        hbr.init(c1);
        hbr.start();
        // get the flow activity entity
        Set<TimelineEntity> entities = hbr.getEntities(new TimelineReaderContext(cluster, null, null, null, null, TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null), new TimelineEntityFilters(10L, null, null, null, null, null, null, null, null), new TimelineDataToRetrieve());
        assertEquals(1, entities.size());
        for (TimelineEntity e : entities) {
            FlowActivityEntity flowActivity = (FlowActivityEntity) e;
            assertEquals(cluster, flowActivity.getCluster());
            assertEquals(user, flowActivity.getUser());
            assertEquals(flow, flowActivity.getFlowName());
            assertEquals(dayTs, Long.valueOf(flowActivity.getDate().getTime()));
            Set<FlowRunEntity> flowRuns = flowActivity.getFlowRuns();
            assertEquals(1, flowRuns.size());
        }
    } finally {
        if (hbr != null) {
            hbr.close();
        }
    }
}

Example 10 with FlowRunEntity

use of org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity in project hadoop by apache.

the class TestHBaseStorageFlowRun method testWriteFlowRunMinMax.

/**
   * Writes 4 timeline entities belonging to one flow run through the
   * {@link HBaseTimelineWriterImpl}
   *
   * Checks the flow run table contents
   *
   * The first entity has a created event, metrics and a finish event.
   *
   * The second entity has a created event and this is the entity with smallest
   * start time. This should be the start time for the flow run.
   *
   * The third entity has a finish event and this is the entity with the max end
   * time. This should be the end time for the flow run.
   *
   * The fourth entity has a created event which has a start time that is
   * greater than min start time.
   *
   */
@Test
public void testWriteFlowRunMinMax() throws Exception {
    TimelineEntities te = new TimelineEntities();
    te.addEntity(TestFlowDataGenerator.getEntity1());
    HBaseTimelineWriterImpl hbi = null;
    Configuration c1 = util.getConfiguration();
    String cluster = "testWriteFlowRunMinMaxToHBase_cluster1";
    String user = "testWriteFlowRunMinMaxToHBase_user1";
    String flow = "testing_flowRun_flow_name";
    String flowVersion = "CF7022C10F1354";
    long runid = 1002345678919L;
    String appName = "application_100000000000_1111";
    long minStartTs = 1425026900000L;
    long greaterStartTs = 30000000000000L;
    long endTs = 1439750690000L;
    TimelineEntity entityMinStartTime = TestFlowDataGenerator.getEntityMinStartTime(minStartTs);
    try {
        hbi = new HBaseTimelineWriterImpl();
        hbi.init(c1);
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // write another entity with the right min start time
        te = new TimelineEntities();
        te.addEntity(entityMinStartTime);
        appName = "application_100000000000_3333";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // writer another entity for max end time
        TimelineEntity entityMaxEndTime = TestFlowDataGenerator.getEntityMaxEndTime(endTs);
        te = new TimelineEntities();
        te.addEntity(entityMaxEndTime);
        appName = "application_100000000000_4444";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // writer another entity with greater start time
        TimelineEntity entityGreaterStartTime = TestFlowDataGenerator.getEntityGreaterStartTime(greaterStartTs);
        te = new TimelineEntities();
        te.addEntity(entityGreaterStartTime);
        appName = "application_1000000000000000_2222";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // flush everything to hbase
        hbi.flush();
    } finally {
        if (hbi != null) {
            hbi.close();
        }
    }
    Connection conn = ConnectionFactory.createConnection(c1);
    // check in flow run table
    Table table1 = conn.getTable(TableName.valueOf(FlowRunTable.DEFAULT_TABLE_NAME));
    // scan the table and see that we get back the right min and max
    // timestamps
    byte[] startRow = new FlowRunRowKey(cluster, user, flow, runid).getRowKey();
    Get g = new Get(startRow);
    g.addColumn(FlowRunColumnFamily.INFO.getBytes(), FlowRunColumn.MIN_START_TIME.getColumnQualifierBytes());
    g.addColumn(FlowRunColumnFamily.INFO.getBytes(), FlowRunColumn.MAX_END_TIME.getColumnQualifierBytes());
    Result r1 = table1.get(g);
    assertNotNull(r1);
    assertTrue(!r1.isEmpty());
    Map<byte[], byte[]> values = r1.getFamilyMap(FlowRunColumnFamily.INFO.getBytes());
    assertEquals(2, r1.size());
    long starttime = Bytes.toLong(values.get(FlowRunColumn.MIN_START_TIME.getColumnQualifierBytes()));
    assertEquals(minStartTs, starttime);
    assertEquals(endTs, Bytes.toLong(values.get(FlowRunColumn.MAX_END_TIME.getColumnQualifierBytes())));
    // use the timeline reader to verify data
    HBaseTimelineReaderImpl hbr = null;
    try {
        hbr = new HBaseTimelineReaderImpl();
        hbr.init(c1);
        hbr.start();
        // get the flow run entity
        TimelineEntity entity = hbr.getEntity(new TimelineReaderContext(cluster, user, flow, runid, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), new TimelineDataToRetrieve());
        assertTrue(TimelineEntityType.YARN_FLOW_RUN.matches(entity.getType()));
        FlowRunEntity flowRun = (FlowRunEntity) entity;
        assertEquals(minStartTs, flowRun.getStartTime());
        assertEquals(endTs, flowRun.getMaxEndTime());
    } finally {
        if (hbr != null) {
            hbr.close();
        }
    }
}