Examples with TimelineEntityFilters org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters used on opensource projects

Example 11 with TimelineEntityFilters

use of org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters in project hadoop by apache.

the class TestHBaseTimelineStorageEntities method testEventsWithEmptyInfo.

@Test
public void testEventsWithEmptyInfo() throws IOException {
    TimelineEvent event = new TimelineEvent();
    String eventId = "foo_ev e  nt_id";
    event.setId(eventId);
    Long expTs = 1436512802000L;
    event.setTimestamp(expTs);
    final TimelineEntity entity = new TimelineEntity();
    entity.setId("attempt_1329348432655_0001_m_000008_18");
    entity.setType("FOO_ATTEMPT");
    entity.addEvent(event);
    TimelineEntities entities = new TimelineEntities();
    entities.addEntity(entity);
    HBaseTimelineWriterImpl hbi = null;
    try {
        Configuration c1 = util.getConfiguration();
        hbi = new HBaseTimelineWriterImpl();
        hbi.init(c1);
        hbi.start();
        String cluster = "cluster_test_empty_eventkey";
        String user = "user_emptyeventkey";
        String flow = "other_flow_name";
        String flowVersion = "1111F01C2287BA";
        long runid = 1009876543218L;
        String appName = ApplicationId.newInstance(System.currentTimeMillis() + 9000000L, 1).toString();
        byte[] startRow = new EntityRowKeyPrefix(cluster, user, flow, runid, appName).getRowKeyPrefix();
        hbi.write(cluster, user, flow, flowVersion, runid, appName, entities);
        hbi.stop();
        // scan the table and see that entity exists
        Scan s = new Scan();
        s.setStartRow(startRow);
        s.addFamily(EntityColumnFamily.INFO.getBytes());
        Connection conn = ConnectionFactory.createConnection(c1);
        ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
        int rowCount = 0;
        for (Result result : scanner) {
            if (result != null && !result.isEmpty()) {
                rowCount++;
                // check the row key
                byte[] row1 = result.getRow();
                assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName, entity));
                Map<EventColumnName, Object> eventsResult = EntityColumnPrefix.EVENT.readResults(result, new EventColumnNameConverter());
                // there should be only one event
                assertEquals(1, eventsResult.size());
                for (Map.Entry<EventColumnName, Object> e : eventsResult.entrySet()) {
                    EventColumnName eventColumnName = e.getKey();
                    // the qualifier is a compound key
                    // hence match individual values
                    assertEquals(eventId, eventColumnName.getId());
                    assertEquals(expTs, eventColumnName.getTimestamp());
                    // key must be empty
                    assertNull(eventColumnName.getInfoKey());
                    Object value = e.getValue();
                    // value should be empty
                    assertEquals("", value.toString());
                }
            }
        }
        assertEquals(1, rowCount);
        // read the timeline entity using the reader this time
        TimelineEntity e1 = reader.getEntity(new TimelineReaderContext(cluster, user, flow, runid, appName, entity.getType(), entity.getId()), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
        Set<TimelineEntity> es1 = reader.getEntities(new TimelineReaderContext(cluster, user, flow, runid, appName, entity.getType(), null), new TimelineEntityFilters(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
        assertNotNull(e1);
        assertEquals(1, es1.size());
        // check the events
        NavigableSet<TimelineEvent> events = e1.getEvents();
        // there should be only one event
        assertEquals(1, events.size());
        for (TimelineEvent e : events) {
            assertEquals(eventId, e.getId());
            assertEquals(expTs, Long.valueOf(e.getTimestamp()));
            Map<String, Object> info = e.getInfo();
            assertTrue(info == null || info.isEmpty());
        }
    } finally {
        if (hbi != null) {
            hbi.stop();
            hbi.close();
        }
    }
}

Example 12 with TimelineEntityFilters

use of org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters in project hadoop by apache.

the class TestHBaseTimelineStorageApps method testReadAppsMetricPrefix.

@Test
public void testReadAppsMetricPrefix() throws Exception {
    TimelineFilterList list = new TimelineFilterList(Operator.OR, new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
    TimelineEntity e1 = reader.getEntity(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1111111111_2222", TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineDataToRetrieve(null, list, null, null));
    assertNotNull(e1);
    assertEquals(1, e1.getMetrics().size());
    Set<TimelineEntity> es1 = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(), new TimelineDataToRetrieve(null, list, null, null));
    int metricCnt = 0;
    for (TimelineEntity entity : es1) {
        metricCnt += entity.getMetrics().size();
        for (TimelineMetric metric : entity.getMetrics()) {
            assertTrue("Metric Id returned should start with MAP1_", metric.getId().startsWith("MAP1_"));
        }
    }
    assertEquals(2, metricCnt);
}

Example 13 with TimelineEntityFilters

use of org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters in project hadoop by apache.

the class TestHBaseTimelineStorageApps method testReadAppsConfigFilters.

@Test
public void testReadAppsConfigFilters() throws Exception {
    TimelineFilterList list1 = new TimelineFilterList();
    list1.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "cfg_param1", "value1"));
    list1.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "cfg_param2", "value2"));
    TimelineFilterList list2 = new TimelineFilterList();
    list2.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "cfg_param1", "value3"));
    list2.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "config_param2", "value2"));
    TimelineFilterList confFilterList = new TimelineFilterList(Operator.OR, list1, list2);
    Set<TimelineEntity> entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, confFilterList, null, null), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), null));
    assertEquals(2, entities.size());
    int cfgCnt = 0;
    for (TimelineEntity entity : entities) {
        cfgCnt += entity.getConfigs().size();
    }
    assertEquals(5, cfgCnt);
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, confFilterList, null, null), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
    assertEquals(2, entities.size());
    cfgCnt = 0;
    for (TimelineEntity entity : entities) {
        cfgCnt += entity.getConfigs().size();
    }
    assertEquals(5, cfgCnt);
    TimelineFilterList confFilterList1 = new TimelineFilterList(new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL, "cfg_param1", "value1"));
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, confFilterList1, null, null), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), null));
    assertEquals(1, entities.size());
    cfgCnt = 0;
    for (TimelineEntity entity : entities) {
        cfgCnt += entity.getConfigs().size();
    }
    assertEquals(3, cfgCnt);
    TimelineFilterList confFilterList2 = new TimelineFilterList(new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL, "cfg_param1", "value1"), new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL, "config_param2", "value2"));
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, confFilterList2, null, null), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), null));
    assertEquals(0, entities.size());
    TimelineFilterList confFilterList3 = new TimelineFilterList(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "dummy_config", "value1"));
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, confFilterList3, null, null), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), null));
    assertEquals(0, entities.size());
    TimelineFilterList confFilterList4 = new TimelineFilterList(new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL, "dummy_config", "value1"));
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, confFilterList4, null, null), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), null));
    assertEquals(0, entities.size());
    TimelineFilterList confFilterList5 = new TimelineFilterList(new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL, "dummy_config", "value1", false));
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, confFilterList5, null, null), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS), null));
    assertEquals(3, entities.size());
}

Example 14 with TimelineEntityFilters

use of org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters in project hadoop by apache.

the class TestHBaseTimelineStorageApps method testReadAppsMetricFilters.

@Test
public void testReadAppsMetricFilters() throws Exception {
    TimelineFilterList list1 = new TimelineFilterList();
    list1.addFilter(new TimelineCompareFilter(TimelineCompareOp.GREATER_OR_EQUAL, "MAP1_SLOT_MILLIS", 50000000900L));
    TimelineFilterList list2 = new TimelineFilterList();
    list2.addFilter(new TimelineCompareFilter(TimelineCompareOp.LESS_THAN, "MAP_SLOT_MILLIS", 80000000000L));
    list2.addFilter(new TimelineCompareFilter(TimelineCompareOp.EQUAL, "MAP1_BYTES", 50));
    TimelineFilterList metricFilterList = new TimelineFilterList(Operator.OR, list1, list2);
    Set<TimelineEntity> entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, null, metricFilterList, null), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), null));
    assertEquals(2, entities.size());
    int metricCnt = 0;
    for (TimelineEntity entity : entities) {
        metricCnt += entity.getMetrics().size();
    }
    assertEquals(3, metricCnt);
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, null, metricFilterList, null), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
    assertEquals(2, entities.size());
    metricCnt = 0;
    for (TimelineEntity entity : entities) {
        metricCnt += entity.getMetrics().size();
    }
    assertEquals(3, metricCnt);
    TimelineFilterList metricFilterList1 = new TimelineFilterList(new TimelineCompareFilter(TimelineCompareOp.LESS_OR_EQUAL, "MAP_SLOT_MILLIS", 80000000000L), new TimelineCompareFilter(TimelineCompareOp.NOT_EQUAL, "MAP1_BYTES", 30));
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, null, metricFilterList1, null), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), null));
    assertEquals(1, entities.size());
    metricCnt = 0;
    for (TimelineEntity entity : entities) {
        metricCnt += entity.getMetrics().size();
    }
    assertEquals(2, metricCnt);
    TimelineFilterList metricFilterList2 = new TimelineFilterList(new TimelineCompareFilter(TimelineCompareOp.LESS_THAN, "MAP_SLOT_MILLIS", 40000000000L), new TimelineCompareFilter(TimelineCompareOp.NOT_EQUAL, "MAP1_BYTES", 30));
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, null, metricFilterList2, null), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), null));
    assertEquals(0, entities.size());
    TimelineFilterList metricFilterList3 = new TimelineFilterList(new TimelineCompareFilter(TimelineCompareOp.EQUAL, "dummy_metric", 5));
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, null, metricFilterList3, null), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), null));
    assertEquals(0, entities.size());
    TimelineFilterList metricFilterList4 = new TimelineFilterList(new TimelineCompareFilter(TimelineCompareOp.NOT_EQUAL, "dummy_metric", 5));
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, null, metricFilterList4, null), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), null));
    assertEquals(0, entities.size());
    TimelineFilterList metricFilterList5 = new TimelineFilterList(new TimelineCompareFilter(TimelineCompareOp.NOT_EQUAL, "dummy_metric", 5, false));
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, null, metricFilterList5, null), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS), null));
    assertEquals(3, entities.size());
}

Example 15 with TimelineEntityFilters

use of org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters in project hadoop by apache.

the class TestHBaseTimelineStorageApps method testReadAppsMetricFilterPrefix.

@Test
public void testReadAppsMetricFilterPrefix() throws Exception {
    TimelineFilterList list = new TimelineFilterList(Operator.OR, new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
    TimelineFilterList metricFilterList = new TimelineFilterList();
    metricFilterList.addFilter(new TimelineCompareFilter(TimelineCompareOp.GREATER_OR_EQUAL, "MAP1_SLOT_MILLIS", 0L));
    Set<TimelineEntity> entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, null, metricFilterList, null), new TimelineDataToRetrieve(null, list, null, null));
    int metricCnt = 0;
    assertEquals(1, entities.size());
    for (TimelineEntity entity : entities) {
        metricCnt += entity.getMetrics().size();
    }
    assertEquals(1, metricCnt);
    TimelineFilterList list1 = new TimelineFilterList();
    list1.addFilter(new TimelineCompareFilter(TimelineCompareOp.GREATER_OR_EQUAL, "MAP1_SLOT_MILLIS", 50000000900L));
    TimelineFilterList list2 = new TimelineFilterList();
    list2.addFilter(new TimelineCompareFilter(TimelineCompareOp.LESS_THAN, "MAP_SLOT_MILLIS", 80000000000L));
    list2.addFilter(new TimelineCompareFilter(TimelineCompareOp.EQUAL, "MAP1_BYTES", 50));
    TimelineFilterList metricsToRetrieve = new TimelineFilterList(Operator.OR, new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
    TimelineFilterList metricFilterList1 = new TimelineFilterList(Operator.OR, list1, list2);
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, null, metricFilterList1, null), new TimelineDataToRetrieve(null, metricsToRetrieve, null, null));
    metricCnt = 0;
    assertEquals(2, entities.size());
    for (TimelineEntity entity : entities) {
        metricCnt += entity.getMetrics().size();
        for (TimelineMetric metric : entity.getMetrics()) {
            assertTrue("Metric Id returned should start with MAP1_", metric.getId().startsWith("MAP1_"));
        }
    }
    assertEquals(2, metricCnt);
    entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, null, metricFilterList1, null), new TimelineDataToRetrieve(null, metricsToRetrieve, EnumSet.of(Field.METRICS), Integer.MAX_VALUE));
    metricCnt = 0;
    int metricValCnt = 0;
    assertEquals(2, entities.size());
    for (TimelineEntity entity : entities) {
        metricCnt += entity.getMetrics().size();
        for (TimelineMetric metric : entity.getMetrics()) {
            metricValCnt += metric.getValues().size();
            assertTrue("Metric Id returned should start with MAP1_", metric.getId().startsWith("MAP1_"));
        }
    }
    assertEquals(2, metricCnt);
    assertEquals(7, metricValCnt);
}