Examples with TimelineDataToRetrieve org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve used on opensource projects

Example 26 with TimelineDataToRetrieve

use of org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve in project hadoop by apache.

the class TestFileSystemTimelineReaderImpl method testGetEntitiesByTimeWindows.

@Test
public void testGetEntitiesByTimeWindows() throws Exception {
    // Get entities based on created time start and end time range.
    Set<TimelineEntity> result = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), new TimelineEntityFilters(null, 1425016502030L, 1425016502060L, null, null, null, null, null, null), new TimelineDataToRetrieve());
    Assert.assertEquals(1, result.size());
    // Only one entity with ID id_4 should be returned.
    for (TimelineEntity entity : result) {
        if (!entity.getId().equals("id_4")) {
            Assert.fail("Incorrect filtering based on created time range");
        }
    }
    // Get entities if only created time end is specified.
    result = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), new TimelineEntityFilters(null, null, 1425016502010L, null, null, null, null, null, null), new TimelineDataToRetrieve());
    Assert.assertEquals(3, result.size());
    for (TimelineEntity entity : result) {
        if (entity.getId().equals("id_4")) {
            Assert.fail("Incorrect filtering based on created time range");
        }
    }
    // Get entities if only created time start is specified.
    result = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), new TimelineEntityFilters(null, 1425016502010L, null, null, null, null, null, null, null), new TimelineDataToRetrieve());
    Assert.assertEquals(1, result.size());
    for (TimelineEntity entity : result) {
        if (!entity.getId().equals("id_4")) {
            Assert.fail("Incorrect filtering based on created time range");
        }
    }
}

Example 27 with TimelineDataToRetrieve

use of org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve in project hadoop by apache.

the class TestFileSystemTimelineReaderImpl method testGetFilteredEntities.

@Test
public void testGetFilteredEntities() throws Exception {
    // Get entities based on info filters.
    TimelineFilterList infoFilterList = new TimelineFilterList();
    infoFilterList.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "info2", 3.5));
    Set<TimelineEntity> result = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), new TimelineEntityFilters(null, null, null, null, null, infoFilterList, null, null, null), new TimelineDataToRetrieve());
    Assert.assertEquals(1, result.size());
    // Only one entity with ID id_3 should be returned.
    for (TimelineEntity entity : result) {
        if (!entity.getId().equals("id_3")) {
            Assert.fail("Incorrect filtering based on info filters");
        }
    }
    // Get entities based on config filters.
    TimelineFilterList confFilterList = new TimelineFilterList();
    confFilterList.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "config_1", "123"));
    confFilterList.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "config_3", "abc"));
    result = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), new TimelineEntityFilters(null, null, null, null, null, null, confFilterList, null, null), new TimelineDataToRetrieve());
    Assert.assertEquals(1, result.size());
    for (TimelineEntity entity : result) {
        if (!entity.getId().equals("id_3")) {
            Assert.fail("Incorrect filtering based on config filters");
        }
    }
    // Get entities based on event filters.
    TimelineFilterList eventFilters = new TimelineFilterList();
    eventFilters.addFilter(new TimelineExistsFilter(TimelineCompareOp.EQUAL, "event_2"));
    eventFilters.addFilter(new TimelineExistsFilter(TimelineCompareOp.EQUAL, "event_4"));
    result = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), new TimelineEntityFilters(null, null, null, null, null, null, null, null, eventFilters), new TimelineDataToRetrieve());
    Assert.assertEquals(1, result.size());
    for (TimelineEntity entity : result) {
        if (!entity.getId().equals("id_3")) {
            Assert.fail("Incorrect filtering based on event filters");
        }
    }
    // Get entities based on metric filters.
    TimelineFilterList metricFilterList = new TimelineFilterList();
    metricFilterList.addFilter(new TimelineCompareFilter(TimelineCompareOp.GREATER_OR_EQUAL, "metric3", 0L));
    result = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), new TimelineEntityFilters(null, null, null, null, null, null, null, metricFilterList, null), new TimelineDataToRetrieve());
    Assert.assertEquals(2, result.size());
    // Two entities with IDs' id_1 and id_2 should be returned.
    for (TimelineEntity entity : result) {
        if (!entity.getId().equals("id_1") && !entity.getId().equals("id_2")) {
            Assert.fail("Incorrect filtering based on metric filters");
        }
    }
    // Get entities based on complex config filters.
    TimelineFilterList list1 = new TimelineFilterList();
    list1.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "config_1", "129"));
    list1.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "config_3", "def"));
    TimelineFilterList list2 = new TimelineFilterList();
    list2.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "config_2", "23"));
    list2.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "config_3", "abc"));
    TimelineFilterList confFilterList1 = new TimelineFilterList(Operator.OR, list1, list2);
    result = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), new TimelineEntityFilters(null, null, null, null, null, null, confFilterList1, null, null), new TimelineDataToRetrieve());
    Assert.assertEquals(2, result.size());
    for (TimelineEntity entity : result) {
        if (!entity.getId().equals("id_1") && !entity.getId().equals("id_2")) {
            Assert.fail("Incorrect filtering based on config filters");
        }
    }
    TimelineFilterList list3 = new TimelineFilterList();
    list3.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL, "config_1", "123"));
    list3.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL, "config_3", "abc"));
    TimelineFilterList list4 = new TimelineFilterList();
    list4.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "config_2", "23"));
    TimelineFilterList confFilterList2 = new TimelineFilterList(Operator.OR, list3, list4);
    result = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), new TimelineEntityFilters(null, null, null, null, null, null, confFilterList2, null, null), new TimelineDataToRetrieve());
    Assert.assertEquals(2, result.size());
    for (TimelineEntity entity : result) {
        if (!entity.getId().equals("id_1") && !entity.getId().equals("id_2")) {
            Assert.fail("Incorrect filtering based on config filters");
        }
    }
    TimelineFilterList confFilterList3 = new TimelineFilterList();
    confFilterList3.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL, "config_1", "127"));
    confFilterList3.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL, "config_3", "abc"));
    result = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), new TimelineEntityFilters(null, null, null, null, null, null, confFilterList3, null, null), new TimelineDataToRetrieve());
    Assert.assertEquals(1, result.size());
    for (TimelineEntity entity : result) {
        if (!entity.getId().equals("id_2")) {
            Assert.fail("Incorrect filtering based on config filters");
        }
    }
    TimelineFilterList confFilterList4 = new TimelineFilterList();
    confFilterList4.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "config_dummy", "dummy"));
    confFilterList4.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "config_3", "def"));
    result = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), new TimelineEntityFilters(null, null, null, null, null, null, confFilterList4, null, null), new TimelineDataToRetrieve());
    Assert.assertEquals(0, result.size());
    TimelineFilterList confFilterList5 = new TimelineFilterList(Operator.OR);
    confFilterList5.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "config_dummy", "dummy"));
    confFilterList5.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "config_3", "def"));
    result = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), new TimelineEntityFilters(null, null, null, null, null, null, confFilterList5, null, null), new TimelineDataToRetrieve());
    Assert.assertEquals(1, result.size());
    for (TimelineEntity entity : result) {
        if (!entity.getId().equals("id_2")) {
            Assert.fail("Incorrect filtering based on config filters");
        }
    }
    // Get entities based on complex metric filters.
    TimelineFilterList list6 = new TimelineFilterList();
    list6.addFilter(new TimelineCompareFilter(TimelineCompareOp.GREATER_THAN, "metric1", 200));
    list6.addFilter(new TimelineCompareFilter(TimelineCompareOp.EQUAL, "metric3", 23));
    TimelineFilterList list7 = new TimelineFilterList();
    list7.addFilter(new TimelineCompareFilter(TimelineCompareOp.GREATER_OR_EQUAL, "metric2", 74));
    TimelineFilterList metricFilterList1 = new TimelineFilterList(Operator.OR, list6, list7);
    result = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), new TimelineEntityFilters(null, null, null, null, null, null, null, metricFilterList1, null), new TimelineDataToRetrieve());
    Assert.assertEquals(2, result.size());
    // Two entities with IDs' id_2 and id_3 should be returned.
    for (TimelineEntity entity : result) {
        if (!entity.getId().equals("id_2") && !entity.getId().equals("id_3")) {
            Assert.fail("Incorrect filtering based on metric filters");
        }
    }
    TimelineFilterList metricFilterList2 = new TimelineFilterList();
    metricFilterList2.addFilter(new TimelineCompareFilter(TimelineCompareOp.LESS_THAN, "metric2", 70));
    metricFilterList2.addFilter(new TimelineCompareFilter(TimelineCompareOp.LESS_OR_EQUAL, "metric3", 23));
    result = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), new TimelineEntityFilters(null, null, null, null, null, null, null, metricFilterList2, null), new TimelineDataToRetrieve());
    Assert.assertEquals(1, result.size());
    for (TimelineEntity entity : result) {
        if (!entity.getId().equals("id_1")) {
            Assert.fail("Incorrect filtering based on metric filters");
        }
    }
    TimelineFilterList metricFilterList3 = new TimelineFilterList();
    metricFilterList3.addFilter(new TimelineCompareFilter(TimelineCompareOp.LESS_THAN, "dummy_metric", 30));
    metricFilterList3.addFilter(new TimelineCompareFilter(TimelineCompareOp.LESS_OR_EQUAL, "metric3", 23));
    result = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), new TimelineEntityFilters(null, null, null, null, null, null, null, metricFilterList3, null), new TimelineDataToRetrieve());
    Assert.assertEquals(0, result.size());
    TimelineFilterList metricFilterList4 = new TimelineFilterList(Operator.OR);
    metricFilterList4.addFilter(new TimelineCompareFilter(TimelineCompareOp.LESS_THAN, "dummy_metric", 30));
    metricFilterList4.addFilter(new TimelineCompareFilter(TimelineCompareOp.LESS_OR_EQUAL, "metric3", 23));
    result = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), new TimelineEntityFilters(null, null, null, null, null, null, null, metricFilterList4, null), new TimelineDataToRetrieve());
    Assert.assertEquals(2, result.size());
    for (TimelineEntity entity : result) {
        if (!entity.getId().equals("id_1") && !entity.getId().equals("id_2")) {
            Assert.fail("Incorrect filtering based on metric filters");
        }
    }
    TimelineFilterList metricFilterList5 = new TimelineFilterList(new TimelineCompareFilter(TimelineCompareOp.NOT_EQUAL, "metric2", 74));
    result = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), new TimelineEntityFilters(null, null, null, null, null, null, null, metricFilterList5, null), new TimelineDataToRetrieve());
    Assert.assertEquals(2, result.size());
    for (TimelineEntity entity : result) {
        if (!entity.getId().equals("id_1") && !entity.getId().equals("id_2")) {
            Assert.fail("Incorrect filtering based on metric filters");
        }
    }
    TimelineFilterList infoFilterList1 = new TimelineFilterList();
    infoFilterList1.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "info2", 3.5));
    infoFilterList1.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL, "info4", 20));
    result = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), new TimelineEntityFilters(null, null, null, null, null, infoFilterList1, null, null, null), new TimelineDataToRetrieve());
    Assert.assertEquals(0, result.size());
    TimelineFilterList infoFilterList2 = new TimelineFilterList(Operator.OR);
    infoFilterList2.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "info2", 3.5));
    infoFilterList2.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "info1", "val1"));
    result = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), new TimelineEntityFilters(null, null, null, null, null, infoFilterList2, null, null, null), new TimelineDataToRetrieve());
    Assert.assertEquals(2, result.size());
    for (TimelineEntity entity : result) {
        if (!entity.getId().equals("id_1") && !entity.getId().equals("id_3")) {
            Assert.fail("Incorrect filtering based on info filters");
        }
    }
    TimelineFilterList infoFilterList3 = new TimelineFilterList();
    infoFilterList3.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "dummy_info", 1));
    infoFilterList3.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "info2", "val5"));
    result = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), new TimelineEntityFilters(null, null, null, null, null, infoFilterList3, null, null, null), new TimelineDataToRetrieve());
    Assert.assertEquals(0, result.size());
    TimelineFilterList infoFilterList4 = new TimelineFilterList(Operator.OR);
    infoFilterList4.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "dummy_info", 1));
    infoFilterList4.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "info2", "val5"));
    result = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), new TimelineEntityFilters(null, null, null, null, null, infoFilterList4, null, null, null), new TimelineDataToRetrieve());
    Assert.assertEquals(1, result.size());
    for (TimelineEntity entity : result) {
        if (!entity.getId().equals("id_1")) {
            Assert.fail("Incorrect filtering based on info filters");
        }
    }
}

Example 28 with TimelineDataToRetrieve

use of org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve in project hadoop by apache.

the class TestFileSystemTimelineReaderImpl method testGetEntityAllFields.

@Test
public void testGetEntityAllFields() throws Exception {
    // All fields of TimelineEntity will be returned.
    TimelineEntity result = reader.getEntity(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", "id_1"), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
    Assert.assertEquals((new TimelineEntity.Identifier("app", "id_1")).toString(), result.getIdentifier().toString());
    Assert.assertEquals((Long) 1425016502000L, result.getCreatedTime());
    Assert.assertEquals(3, result.getConfigs().size());
    Assert.assertEquals(3, result.getMetrics().size());
    // All fields including events will be returned.
    Assert.assertEquals(2, result.getEvents().size());
}

Example 29 with TimelineDataToRetrieve

use of org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve in project hadoop by apache.

the class TestFileSystemTimelineReaderImpl method testGetAllEntities.

@Test
public void testGetAllEntities() throws Exception {
    Set<TimelineEntity> result = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1", "app", null), new TimelineEntityFilters(), new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
    // All 4 entities will be returned
    Assert.assertEquals(4, result.size());
}

Example 30 with TimelineDataToRetrieve

use of org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve in project hadoop by apache.

the class FlowRunEntityReader method constructFilterListBasedOnFields.

@Override
protected FilterList constructFilterListBasedOnFields() throws IOException {
    FilterList list = new FilterList(Operator.MUST_PASS_ONE);
    // By default fetch everything in INFO column family.
    FamilyFilter infoColumnFamily = new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(FlowRunColumnFamily.INFO.getBytes()));
    TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve();
    // Metrics are always returned if we are reading a single entity.
    if (!isSingleEntityRead() && !hasField(dataToRetrieve.getFieldsToRetrieve(), Field.METRICS)) {
        FilterList infoColFamilyList = new FilterList(Operator.MUST_PASS_ONE);
        infoColFamilyList.addFilter(infoColumnFamily);
        infoColFamilyList.addFilter(new QualifierFilter(CompareOp.NOT_EQUAL, new BinaryPrefixComparator(FlowRunColumnPrefix.METRIC.getColumnPrefixBytes(""))));
        list.addFilter(infoColFamilyList);
    } else {
        // Check if metricsToRetrieve are specified and if they are, create a
        // filter list for info column family by adding flow run tables columns
        // and a list for metrics to retrieve. Pls note that fieldsToRetrieve
        // will have METRICS added to it if metricsToRetrieve are specified
        // (in augmentParams()).
        TimelineFilterList metricsToRetrieve = dataToRetrieve.getMetricsToRetrieve();
        if (metricsToRetrieve != null && !metricsToRetrieve.getFilterList().isEmpty()) {
            FilterList infoColFamilyList = new FilterList();
            infoColFamilyList.addFilter(infoColumnFamily);
            FilterList columnsList = updateFixedColumns();
            columnsList.addFilter(TimelineFilterUtils.createHBaseFilterList(FlowRunColumnPrefix.METRIC, metricsToRetrieve));
            infoColFamilyList.addFilter(columnsList);
            list.addFilter(infoColFamilyList);
        }
    }
    return list;
}