Examples with HBaseTimelineWriterImpl org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl used on opensource projects

Example 6 with HBaseTimelineWriterImpl

use of org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl in project hadoop by apache.

the class TestHBaseStorageFlowRunCompaction method testWriteFlowRunCompaction.

@Test
public void testWriteFlowRunCompaction() throws Exception {
    String cluster = "kompaction_cluster1";
    String user = "kompaction_FlowRun__user1";
    String flow = "kompaction_flowRun_flow_name";
    String flowVersion = "AF1021C19F1351";
    long runid = 1449526652000L;
    int start = 10;
    int count = 2000;
    int appIdSuffix = 1;
    HBaseTimelineWriterImpl hbi = null;
    long insertTs = System.currentTimeMillis() - count;
    Configuration c1 = util.getConfiguration();
    TimelineEntities te1 = null;
    TimelineEntity entityApp1 = null;
    try {
        hbi = new HBaseTimelineWriterImpl();
        hbi.init(c1);
        // of metric1 and 100 of metric2
        for (int i = start; i < start + count; i++) {
            String appName = "application_10240000000000_" + appIdSuffix;
            insertTs++;
            te1 = new TimelineEntities();
            entityApp1 = TestFlowDataGenerator.getEntityMetricsApp1(insertTs, c1);
            te1.addEntity(entityApp1);
            hbi.write(cluster, user, flow, flowVersion, runid, appName, te1);
            appName = "application_2048000000000_7" + appIdSuffix;
            insertTs++;
            te1 = new TimelineEntities();
            entityApp1 = TestFlowDataGenerator.getEntityMetricsApp2(insertTs);
            te1.addEntity(entityApp1);
            hbi.write(cluster, user, flow, flowVersion, runid, appName, te1);
        }
    } finally {
        String appName = "application_10240000000000_" + appIdSuffix;
        te1 = new TimelineEntities();
        entityApp1 = TestFlowDataGenerator.getEntityMetricsApp1Complete(insertTs + 1, c1);
        te1.addEntity(entityApp1);
        if (hbi != null) {
            hbi.write(cluster, user, flow, flowVersion, runid, appName, te1);
            hbi.flush();
            hbi.close();
        }
    }
    // check in flow run table
    HRegionServer server = util.getRSForFirstRegionInTable(TableName.valueOf(FlowRunTable.DEFAULT_TABLE_NAME));
    List<Region> regions = server.getOnlineRegions(TableName.valueOf(FlowRunTable.DEFAULT_TABLE_NAME));
    assertTrue("Didn't find any regions for primary table!", regions.size() > 0);
    // flush and compact all the regions of the primary table
    for (Region region : regions) {
        region.flush(true);
        region.compact(true);
    }
    // check flow run for one flow many apps
    checkFlowRunTable(cluster, user, flow, runid, c1, 4);
}

Example 7 with HBaseTimelineWriterImpl

use of org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl in project hadoop by apache.

the class TestHBaseStorageFlowActivity method testWriteFlowRunMinMax.

/**
   * Writes 4 timeline entities belonging to one flow run through the
   * {@link HBaseTimelineWriterImpl}
   *
   * Checks the flow run table contents
   *
   * The first entity has a created event, metrics and a finish event.
   *
   * The second entity has a created event and this is the entity with smallest
   * start time. This should be the start time for the flow run.
   *
   * The third entity has a finish event and this is the entity with the max end
   * time. This should be the end time for the flow run.
   *
   * The fourth entity has a created event which has a start time that is
   * greater than min start time.
   *
   * The test also checks in the flow activity table that one entry has been
   * made for all of these 4 application entities since they belong to the same
   * flow run.
   */
@Test
public void testWriteFlowRunMinMax() throws Exception {
    TimelineEntities te = new TimelineEntities();
    te.addEntity(TestFlowDataGenerator.getEntity1());
    HBaseTimelineWriterImpl hbi = null;
    Configuration c1 = util.getConfiguration();
    String cluster = "testWriteFlowRunMinMaxToHBase_cluster1";
    String user = "testWriteFlowRunMinMaxToHBase_user1";
    String flow = "testing_flowRun_flow_name";
    String flowVersion = "CF7022C10F1354";
    long runid = 1002345678919L;
    String appName = "application_100000000000_1111";
    long minStartTs = 1424995200300L;
    long greaterStartTs = 1424995200300L + 864000L;
    long endTs = 1424995200300L + 86000000L;
    TimelineEntity entityMinStartTime = TestFlowDataGenerator.getEntityMinStartTime(minStartTs);
    try {
        hbi = new HBaseTimelineWriterImpl();
        hbi.init(c1);
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // write another entity with the right min start time
        te = new TimelineEntities();
        te.addEntity(entityMinStartTime);
        appName = "application_100000000000_3333";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // writer another entity for max end time
        TimelineEntity entityMaxEndTime = TestFlowDataGenerator.getEntityMaxEndTime(endTs);
        te = new TimelineEntities();
        te.addEntity(entityMaxEndTime);
        appName = "application_100000000000_4444";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // writer another entity with greater start time
        TimelineEntity entityGreaterStartTime = TestFlowDataGenerator.getEntityGreaterStartTime(greaterStartTs);
        te = new TimelineEntities();
        te.addEntity(entityGreaterStartTime);
        appName = "application_1000000000000000_2222";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // flush everything to hbase
        hbi.flush();
    } finally {
        if (hbi != null) {
            hbi.close();
        }
    }
    Connection conn = ConnectionFactory.createConnection(c1);
    // check in flow activity table
    Table table1 = conn.getTable(TableName.valueOf(FlowActivityTable.DEFAULT_TABLE_NAME));
    byte[] startRow = new FlowActivityRowKey(cluster, minStartTs, user, flow).getRowKey();
    Get g = new Get(startRow);
    Result r1 = table1.get(g);
    assertNotNull(r1);
    assertTrue(!r1.isEmpty());
    Map<byte[], byte[]> values = r1.getFamilyMap(FlowActivityColumnFamily.INFO.getBytes());
    assertEquals(1, values.size());
    byte[] row = r1.getRow();
    FlowActivityRowKey flowActivityRowKey = FlowActivityRowKey.parseRowKey(row);
    assertNotNull(flowActivityRowKey);
    assertEquals(cluster, flowActivityRowKey.getClusterId());
    assertEquals(user, flowActivityRowKey.getUserId());
    assertEquals(flow, flowActivityRowKey.getFlowName());
    Long dayTs = HBaseTimelineStorageUtils.getTopOfTheDayTimestamp(minStartTs);
    assertEquals(dayTs, flowActivityRowKey.getDayTimestamp());
    assertEquals(1, values.size());
    checkFlowActivityRunId(runid, flowVersion, values);
    // use the timeline reader to verify data
    HBaseTimelineReaderImpl hbr = null;
    try {
        hbr = new HBaseTimelineReaderImpl();
        hbr.init(c1);
        hbr.start();
        // get the flow activity entity
        Set<TimelineEntity> entities = hbr.getEntities(new TimelineReaderContext(cluster, null, null, null, null, TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null), new TimelineEntityFilters(10L, null, null, null, null, null, null, null, null), new TimelineDataToRetrieve());
        assertEquals(1, entities.size());
        for (TimelineEntity e : entities) {
            FlowActivityEntity flowActivity = (FlowActivityEntity) e;
            assertEquals(cluster, flowActivity.getCluster());
            assertEquals(user, flowActivity.getUser());
            assertEquals(flow, flowActivity.getFlowName());
            assertEquals(dayTs, Long.valueOf(flowActivity.getDate().getTime()));
            Set<FlowRunEntity> flowRuns = flowActivity.getFlowRuns();
            assertEquals(1, flowRuns.size());
        }
    } finally {
        if (hbr != null) {
            hbr.close();
        }
    }
}

Example 8 with HBaseTimelineWriterImpl

use of org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl in project hadoop by apache.

the class TestHBaseStorageFlowRun method testWriteFlowRunMetricsOneFlow.

/**
   * Writes two application entities of the same flow run. Each application has
   * two metrics: slot millis and hdfs bytes read. Each metric has values at two
   * timestamps.
   *
   * Checks the metric values of the flow in the flow run table. Flow metric
   * values should be the sum of individual metric values that belong to the
   * latest timestamp for that metric
   */
@Test
public void testWriteFlowRunMetricsOneFlow() throws Exception {
    String cluster = "testWriteFlowRunMetricsOneFlow_cluster1";
    String user = "testWriteFlowRunMetricsOneFlow_user1";
    String flow = "testing_flowRun_metrics_flow_name";
    String flowVersion = "CF7022C10F1354";
    long runid = 1002345678919L;
    TimelineEntities te = new TimelineEntities();
    TimelineEntity entityApp1 = TestFlowDataGenerator.getEntityMetricsApp1(System.currentTimeMillis());
    te.addEntity(entityApp1);
    HBaseTimelineWriterImpl hbi = null;
    Configuration c1 = util.getConfiguration();
    try {
        hbi = new HBaseTimelineWriterImpl();
        hbi.init(c1);
        String appName = "application_11111111111111_1111";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        // write another application with same metric to this flow
        te = new TimelineEntities();
        TimelineEntity entityApp2 = TestFlowDataGenerator.getEntityMetricsApp2(System.currentTimeMillis());
        te.addEntity(entityApp2);
        appName = "application_11111111111111_2222";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        hbi.flush();
    } finally {
        if (hbi != null) {
            hbi.close();
        }
    }
    // check flow run
    checkFlowRunTable(cluster, user, flow, runid, c1);
    // check various batch limits in scanning the table for this flow
    checkFlowRunTableBatchLimit(cluster, user, flow, runid, c1);
    // use the timeline reader to verify data
    HBaseTimelineReaderImpl hbr = null;
    try {
        hbr = new HBaseTimelineReaderImpl();
        hbr.init(c1);
        hbr.start();
        TimelineEntity entity = hbr.getEntity(new TimelineReaderContext(cluster, user, flow, runid, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), new TimelineDataToRetrieve());
        assertTrue(TimelineEntityType.YARN_FLOW_RUN.matches(entity.getType()));
        Set<TimelineMetric> metrics = entity.getMetrics();
        assertEquals(2, metrics.size());
        for (TimelineMetric metric : metrics) {
            String id = metric.getId();
            Map<Long, Number> values = metric.getValues();
            assertEquals(1, values.size());
            Number value = null;
            for (Number n : values.values()) {
                value = n;
            }
            switch(id) {
                case METRIC1:
                    assertEquals(141L, value);
                    break;
                case METRIC2:
                    assertEquals(57L, value);
                    break;
                default:
                    fail("unrecognized metric: " + id);
            }
        }
    } finally {
        if (hbr != null) {
            hbr.close();
        }
    }
}

Example 9 with HBaseTimelineWriterImpl

use of org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl in project hadoop by apache.

the class TestHBaseStorageFlowRun method testFilterFlowRunsByCreatedTime.

@Test
public void testFilterFlowRunsByCreatedTime() throws Exception {
    String cluster = "cluster2";
    String user = "user2";
    String flow = "flow_name2";
    TimelineEntities te = new TimelineEntities();
    TimelineEntity entityApp1 = TestFlowDataGenerator.getEntityMetricsApp1(System.currentTimeMillis());
    entityApp1.setCreatedTime(1425016501000L);
    te.addEntity(entityApp1);
    HBaseTimelineWriterImpl hbi = null;
    Configuration c1 = util.getConfiguration();
    try {
        hbi = new HBaseTimelineWriterImpl();
        hbi.init(c1);
        hbi.write(cluster, user, flow, "CF7022C10F1354", 1002345678919L, "application_11111111111111_1111", te);
        // write another application with same metric to this flow
        te = new TimelineEntities();
        TimelineEntity entityApp2 = TestFlowDataGenerator.getEntityMetricsApp2(System.currentTimeMillis());
        entityApp2.setCreatedTime(1425016502000L);
        te.addEntity(entityApp2);
        hbi.write(cluster, user, flow, "CF7022C10F1354", 1002345678918L, "application_11111111111111_2222", te);
        hbi.flush();
    } finally {
        if (hbi != null) {
            hbi.close();
        }
    }
    // use the timeline reader to verify data
    HBaseTimelineReaderImpl hbr = null;
    try {
        hbr = new HBaseTimelineReaderImpl();
        hbr.init(c1);
        hbr.start();
        Set<TimelineEntity> entities = hbr.getEntities(new TimelineReaderContext(cluster, user, flow, null, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), new TimelineEntityFilters(null, 1425016501000L, 1425016502001L, null, null, null, null, null, null), new TimelineDataToRetrieve());
        assertEquals(2, entities.size());
        for (TimelineEntity entity : entities) {
            if (!entity.getId().equals("user2@flow_name2/1002345678918") && !entity.getId().equals("user2@flow_name2/1002345678919")) {
                fail("Entities with flow runs 1002345678918 and 1002345678919" + "should be present.");
            }
        }
        entities = hbr.getEntities(new TimelineReaderContext(cluster, user, flow, null, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), new TimelineEntityFilters(null, 1425016501050L, null, null, null, null, null, null, null), new TimelineDataToRetrieve());
        assertEquals(1, entities.size());
        for (TimelineEntity entity : entities) {
            if (!entity.getId().equals("user2@flow_name2/1002345678918")) {
                fail("Entity with flow run 1002345678918 should be present.");
            }
        }
        entities = hbr.getEntities(new TimelineReaderContext(cluster, user, flow, null, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), new TimelineEntityFilters(null, null, 1425016501050L, null, null, null, null, null, null), new TimelineDataToRetrieve());
        assertEquals(1, entities.size());
        for (TimelineEntity entity : entities) {
            if (!entity.getId().equals("user2@flow_name2/1002345678919")) {
                fail("Entity with flow run 1002345678919 should be present.");
            }
        }
    } finally {
        if (hbr != null) {
            hbr.close();
        }
    }
}

Example 10 with HBaseTimelineWriterImpl

use of org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl in project hadoop by apache.

the class TestHBaseStorageFlowRun method testWriteFlowRunFlush.

@Test
public void testWriteFlowRunFlush() throws Exception {
    String cluster = "atestFlushFlowRun_cluster1";
    String user = "atestFlushFlowRun__user1";
    String flow = "atestFlushFlowRun_flow_name";
    String flowVersion = "AF1021C19F1351";
    long runid = 1449526652000L;
    int start = 10;
    int count = 20000;
    int appIdSuffix = 1;
    HBaseTimelineWriterImpl hbi = null;
    long insertTs = 1449796654827L - count;
    long minTS = insertTs + 1;
    long startTs = insertTs;
    Configuration c1 = util.getConfiguration();
    TimelineEntities te1 = null;
    TimelineEntity entityApp1 = null;
    TimelineEntity entityApp2 = null;
    try {
        hbi = new HBaseTimelineWriterImpl();
        hbi.init(c1);
        for (int i = start; i < count; i++) {
            String appName = "application_1060350000000_" + appIdSuffix;
            insertTs++;
            te1 = new TimelineEntities();
            entityApp1 = TestFlowDataGenerator.getMinFlushEntity(insertTs);
            te1.addEntity(entityApp1);
            entityApp2 = TestFlowDataGenerator.getMaxFlushEntity(insertTs);
            te1.addEntity(entityApp2);
            hbi.write(cluster, user, flow, flowVersion, runid, appName, te1);
            Thread.sleep(1);
            appName = "application_1001199480000_7" + appIdSuffix;
            insertTs++;
            appIdSuffix++;
            te1 = new TimelineEntities();
            entityApp1 = TestFlowDataGenerator.getMinFlushEntity(insertTs);
            te1.addEntity(entityApp1);
            entityApp2 = TestFlowDataGenerator.getMaxFlushEntity(insertTs);
            te1.addEntity(entityApp2);
            hbi.write(cluster, user, flow, flowVersion, runid, appName, te1);
            if (i % 1000 == 0) {
                hbi.flush();
                checkMinMaxFlush(c1, minTS, startTs, count, cluster, user, flow, runid, false);
            }
        }
    } finally {
        if (hbi != null) {
            hbi.flush();
            hbi.close();
        }
        checkMinMaxFlush(c1, minTS, startTs, count, cluster, user, flow, runid, true);
    }
}