use of org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList in project hadoop by apache.
the class TestHBaseTimelineStorageApps method testReadAppsMetricFilterPrefix.
@Test
public void testReadAppsMetricFilterPrefix() throws Exception {
TimelineFilterList list = new TimelineFilterList(Operator.OR, new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
TimelineFilterList metricFilterList = new TimelineFilterList();
metricFilterList.addFilter(new TimelineCompareFilter(TimelineCompareOp.GREATER_OR_EQUAL, "MAP1_SLOT_MILLIS", 0L));
Set<TimelineEntity> entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, null, metricFilterList, null), new TimelineDataToRetrieve(null, list, null, null));
int metricCnt = 0;
assertEquals(1, entities.size());
for (TimelineEntity entity : entities) {
metricCnt += entity.getMetrics().size();
}
assertEquals(1, metricCnt);
TimelineFilterList list1 = new TimelineFilterList();
list1.addFilter(new TimelineCompareFilter(TimelineCompareOp.GREATER_OR_EQUAL, "MAP1_SLOT_MILLIS", 50000000900L));
TimelineFilterList list2 = new TimelineFilterList();
list2.addFilter(new TimelineCompareFilter(TimelineCompareOp.LESS_THAN, "MAP_SLOT_MILLIS", 80000000000L));
list2.addFilter(new TimelineCompareFilter(TimelineCompareOp.EQUAL, "MAP1_BYTES", 50));
TimelineFilterList metricsToRetrieve = new TimelineFilterList(Operator.OR, new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
TimelineFilterList metricFilterList1 = new TimelineFilterList(Operator.OR, list1, list2);
entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, null, metricFilterList1, null), new TimelineDataToRetrieve(null, metricsToRetrieve, null, null));
metricCnt = 0;
assertEquals(2, entities.size());
for (TimelineEntity entity : entities) {
metricCnt += entity.getMetrics().size();
for (TimelineMetric metric : entity.getMetrics()) {
assertTrue("Metric Id returned should start with MAP1_", metric.getId().startsWith("MAP1_"));
}
}
assertEquals(2, metricCnt);
entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(), null), new TimelineEntityFilters(null, null, null, null, null, null, null, metricFilterList1, null), new TimelineDataToRetrieve(null, metricsToRetrieve, EnumSet.of(Field.METRICS), Integer.MAX_VALUE));
metricCnt = 0;
int metricValCnt = 0;
assertEquals(2, entities.size());
for (TimelineEntity entity : entities) {
metricCnt += entity.getMetrics().size();
for (TimelineMetric metric : entity.getMetrics()) {
metricValCnt += metric.getValues().size();
assertTrue("Metric Id returned should start with MAP1_", metric.getId().startsWith("MAP1_"));
}
}
assertEquals(2, metricCnt);
assertEquals(7, metricValCnt);
}
use of org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList in project hadoop by apache.
the class TestHBaseTimelineStorageEntities method testReadEntitiesConfigPrefix.
@Test
public void testReadEntitiesConfigPrefix() throws Exception {
TimelineFilterList list = new TimelineFilterList(Operator.OR, new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "cfg_"));
TimelineEntity e1 = reader.getEntity(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", "hello"), new TimelineDataToRetrieve(list, null, null, null));
assertNotNull(e1);
assertEquals(1, e1.getConfigs().size());
Set<TimelineEntity> es1 = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), new TimelineEntityFilters(), new TimelineDataToRetrieve(list, null, null, null));
int cfgCnt = 0;
for (TimelineEntity entity : es1) {
cfgCnt += entity.getConfigs().size();
for (String confKey : entity.getConfigs().keySet()) {
assertTrue("Config key returned should start with cfg_", confKey.startsWith("cfg_"));
}
}
assertEquals(3, cfgCnt);
}
use of org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList in project hadoop by apache.
the class TestHBaseTimelineStorageEntities method testReadEntitiesConfigFilterPrefix.
@Test
public void testReadEntitiesConfigFilterPrefix() throws Exception {
TimelineFilterList confFilterList = new TimelineFilterList();
confFilterList.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "cfg_param1", "value1"));
TimelineFilterList list = new TimelineFilterList(Operator.OR, new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "cfg_"));
Set<TimelineEntity> entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), new TimelineEntityFilters(null, null, null, null, null, null, confFilterList, null, null), new TimelineDataToRetrieve(list, null, null, null));
assertEquals(1, entities.size());
int cfgCnt = 0;
for (TimelineEntity entity : entities) {
cfgCnt += entity.getConfigs().size();
for (String confKey : entity.getConfigs().keySet()) {
assertTrue("Config key returned should start with cfg_", confKey.startsWith("cfg_"));
}
}
assertEquals(2, cfgCnt);
TimelineFilterList list1 = new TimelineFilterList();
list1.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "cfg_param1", "value1"));
list1.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "cfg_param2", "value2"));
TimelineFilterList list2 = new TimelineFilterList();
list2.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "cfg_param1", "value3"));
list2.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "config_param2", "value2"));
TimelineFilterList confFilterList1 = new TimelineFilterList(Operator.OR, list1, list2);
TimelineFilterList confsToRetrieve = new TimelineFilterList(Operator.OR, new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "config_"));
entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1", "some_flow_name", 1002345678919L, "application_1231111111_1111", "world", null), new TimelineEntityFilters(null, null, null, null, null, null, confFilterList1, null, null), new TimelineDataToRetrieve(confsToRetrieve, null, null, null));
assertEquals(2, entities.size());
cfgCnt = 0;
for (TimelineEntity entity : entities) {
cfgCnt += entity.getConfigs().size();
for (String confKey : entity.getConfigs().keySet()) {
assertTrue("Config key returned should start with config_", confKey.startsWith("config_"));
}
}
assertEquals(2, cfgCnt);
}
use of org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList in project hadoop by apache.
the class TestHBaseStorageFlowRun method testWriteFlowRunMetricsPrefix.
@Test
public void testWriteFlowRunMetricsPrefix() throws Exception {
String cluster = "testWriteFlowRunMetricsPrefix_cluster1";
String user = "testWriteFlowRunMetricsPrefix_user1";
String flow = "testWriteFlowRunMetricsPrefix_flow_name";
String flowVersion = "CF7022C10F1354";
TimelineEntities te = new TimelineEntities();
TimelineEntity entityApp1 = TestFlowDataGenerator.getEntityMetricsApp1(System.currentTimeMillis());
te.addEntity(entityApp1);
HBaseTimelineWriterImpl hbi = null;
Configuration c1 = util.getConfiguration();
try {
hbi = new HBaseTimelineWriterImpl();
hbi.init(c1);
String appName = "application_11111111111111_1111";
hbi.write(cluster, user, flow, flowVersion, 1002345678919L, appName, te);
// write another application with same metric to this flow
te = new TimelineEntities();
TimelineEntity entityApp2 = TestFlowDataGenerator.getEntityMetricsApp2(System.currentTimeMillis());
te.addEntity(entityApp2);
appName = "application_11111111111111_2222";
hbi.write(cluster, user, flow, flowVersion, 1002345678918L, appName, te);
hbi.flush();
} finally {
if (hbi != null) {
hbi.close();
}
}
// use the timeline reader to verify data
HBaseTimelineReaderImpl hbr = null;
try {
hbr = new HBaseTimelineReaderImpl();
hbr.init(c1);
hbr.start();
TimelineFilterList metricsToRetrieve = new TimelineFilterList(Operator.OR, new TimelinePrefixFilter(TimelineCompareOp.EQUAL, METRIC1.substring(0, METRIC1.indexOf("_") + 1)));
TimelineEntity entity = hbr.getEntity(new TimelineReaderContext(cluster, user, flow, 1002345678919L, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), new TimelineDataToRetrieve(null, metricsToRetrieve, null, null));
assertTrue(TimelineEntityType.YARN_FLOW_RUN.matches(entity.getType()));
Set<TimelineMetric> metrics = entity.getMetrics();
assertEquals(1, metrics.size());
for (TimelineMetric metric : metrics) {
String id = metric.getId();
Map<Long, Number> values = metric.getValues();
assertEquals(1, values.size());
Number value = null;
for (Number n : values.values()) {
value = n;
}
switch(id) {
case METRIC1:
assertEquals(40L, value);
break;
default:
fail("unrecognized metric: " + id);
}
}
Set<TimelineEntity> entities = hbr.getEntities(new TimelineReaderContext(cluster, user, flow, null, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), new TimelineEntityFilters(), new TimelineDataToRetrieve(null, metricsToRetrieve, null, null));
assertEquals(2, entities.size());
int metricCnt = 0;
for (TimelineEntity timelineEntity : entities) {
metricCnt += timelineEntity.getMetrics().size();
}
assertEquals(2, metricCnt);
} finally {
if (hbr != null) {
hbr.close();
}
}
}
use of org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList in project hadoop by apache.
the class TimelineParserForDataToRetrieve method parse.
@Override
public TimelineFilterList parse() throws TimelineParseException {
if (expr == null || exprLength == 0) {
return null;
}
TimelineCompareOp compareOp = null;
int openingBracketIndex = expr.indexOf(TimelineParseConstants.OPENING_BRACKET_CHAR);
if (expr.charAt(0) == TimelineParseConstants.NOT_CHAR) {
if (openingBracketIndex == -1) {
throw new TimelineParseException("Invalid config/metric to retrieve " + "expression");
}
if (openingBracketIndex != 1 && expr.substring(1, openingBracketIndex + 1).trim().length() != 1) {
throw new TimelineParseException("Invalid config/metric to retrieve " + "expression");
}
compareOp = TimelineCompareOp.NOT_EQUAL;
} else if (openingBracketIndex <= 0) {
compareOp = TimelineCompareOp.EQUAL;
}
char lastChar = expr.charAt(exprLength - 1);
if (compareOp == TimelineCompareOp.NOT_EQUAL && lastChar != TimelineParseConstants.CLOSING_BRACKET_CHAR) {
throw new TimelineParseException("Invalid config/metric to retrieve " + "expression");
}
if (openingBracketIndex != -1 && expr.charAt(exprLength - 1) == TimelineParseConstants.CLOSING_BRACKET_CHAR) {
expr = expr.substring(openingBracketIndex + 1, exprLength - 1).trim();
}
if (expr.isEmpty()) {
return null;
}
Operator op = (compareOp == TimelineCompareOp.NOT_EQUAL) ? Operator.AND : Operator.OR;
TimelineFilterList list = new TimelineFilterList(op);
String[] splits = expr.split(TimelineParseConstants.COMMA_DELIMITER);
for (String split : splits) {
list.addFilter(new TimelinePrefixFilter(compareOp, split.trim()));
}
return list;
}
Aggregations