use of org.hbase.async.KeyRegexpFilter in project opentsdb by OpenTSDB.
the class TestTsdbQueryQueries method filterExplicitTagsGroupByOK.
@Test
public void filterExplicitTagsGroupByOK() throws Exception {
tsdb.getConfig().overrideConfig("tsd.query.enable_fuzzy", "true");
storeLongTimeSeriesSeconds(true, false);
HashMap<String, String> tags = new HashMap<String, String>(1);
tags.put("host", "*");
query.setStartTime(1356998400);
query.setEndTime(1357041600);
query.setExplicitTags(true);
query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false);
final DataPoints[] dps = query.run();
assertNotNull(dps);
assertEquals("sys.cpu.user", dps[0].metricName());
assertTrue(dps[0].getAggregatedTags().isEmpty());
assertNull(dps[0].getAnnotations());
assertEquals("web01", dps[0].getTags().get("host"));
int value = 1;
for (DataPoint dp : dps[0]) {
assertEquals(value, dp.longValue());
value++;
}
assertEquals(300, dps[0].aggregatedSize());
// assert fuzzy
for (final MockScanner scanner : storage.getScanners()) {
assertTrue(scanner.getFilter() instanceof FilterList);
FilterList filter_list = (FilterList) scanner.getFilter();
assertEquals(2, filter_list.size());
assertTrue(filter_list.filters().get(0) instanceof FuzzyRowFilter);
assertTrue(filter_list.filters().get(1) instanceof KeyRegexpFilter);
}
}
use of org.hbase.async.KeyRegexpFilter in project opentsdb by OpenTSDB.
the class TestTsdbQueryQueries method filterExplicitTagsMissing.
@Test
public void filterExplicitTagsMissing() throws Exception {
tsdb.getConfig().overrideConfig("tsd.query.enable_fuzzy", "true");
when(tag_names.getIdAsync("colo")).thenReturn(Deferred.fromResult(new byte[] { 0, 0, 0, 4 }));
when(tag_values.getIdAsync("lga")).thenReturn(Deferred.fromResult(new byte[] { 0, 0, 0, 4 }));
storeLongTimeSeriesSeconds(true, false);
HashMap<String, String> tags = new HashMap<String, String>(1);
tags.put("host", "web01");
tags.put("colo", "lga");
query.setStartTime(1356998400);
query.setEndTime(1357041600);
query.setExplicitTags(true);
query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false);
final DataPoints[] dps = query.run();
assertNotNull(dps);
assertEquals(0, dps.length);
// assert fuzzy
for (final MockScanner scanner : storage.getScanners()) {
assertTrue(scanner.getFilter() instanceof FilterList);
FilterList filter_list = (FilterList) scanner.getFilter();
assertEquals(2, filter_list.size());
assertTrue(filter_list.filters().get(0) instanceof FuzzyRowFilter);
assertTrue(filter_list.filters().get(1) instanceof KeyRegexpFilter);
}
}
use of org.hbase.async.KeyRegexpFilter in project opentsdb by OpenTSDB.
the class TestTsdbQueryQueries method filterExplicitTagsOK.
@Test
public void filterExplicitTagsOK() throws Exception {
tsdb.getConfig().overrideConfig("tsd.query.enable_fuzzy", "true");
storeLongTimeSeriesSeconds(true, false);
HashMap<String, String> tags = new HashMap<String, String>(1);
tags.put("host", "web01");
query.setStartTime(1356998400);
query.setEndTime(1357041600);
query.setExplicitTags(true);
query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false);
final DataPoints[] dps = query.run();
assertNotNull(dps);
assertEquals("sys.cpu.user", dps[0].metricName());
assertTrue(dps[0].getAggregatedTags().isEmpty());
assertNull(dps[0].getAnnotations());
assertEquals("web01", dps[0].getTags().get("host"));
int value = 1;
for (DataPoint dp : dps[0]) {
assertEquals(value, dp.longValue());
value++;
}
assertEquals(300, dps[0].aggregatedSize());
// assert fuzzy
for (final MockScanner scanner : storage.getScanners()) {
assertTrue(scanner.getFilter() instanceof FilterList);
FilterList filter_list = (FilterList) scanner.getFilter();
assertEquals(2, filter_list.size());
assertTrue(filter_list.filters().get(0) instanceof FuzzyRowFilter);
assertTrue(filter_list.filters().get(1) instanceof KeyRegexpFilter);
}
}
use of org.hbase.async.KeyRegexpFilter in project opentsdb by OpenTSDB.
the class QueryUtil method setDataTableScanFilter.
/**
* Sets a filter or filter list on the scanner based on whether or not the
* query had tags it needed to match.
* NOTE: This method will sort the group bys.
* @param scanner The scanner to modify.
* @param group_bys An optional list of tag keys that we want to group on. May
* be null.
* @param row_key_literals An optional list of key value pairs to filter on.
* May be null.
* @param explicit_tags Whether or not explicit tags are enabled so that the
* regex only picks out series with the specified tags
* @param enable_fuzzy_filter Whether or not a fuzzy filter should be used
* in combination with the explicit tags param. If explicit tags is disabled
* then this param is ignored.
* @param end_time The end of the query time so the fuzzy filter knows when
* to stop scanning.
*/
public static void setDataTableScanFilter(final Scanner scanner, final List<byte[]> group_bys, final ByteMap<byte[][]> row_key_literals, final boolean explicit_tags, final boolean enable_fuzzy_filter, final int end_time) {
// no-op
if ((group_bys == null || group_bys.isEmpty()) && (row_key_literals == null || row_key_literals.isEmpty())) {
return;
}
if (group_bys != null) {
Collections.sort(group_bys, Bytes.MEMCMP);
}
final int prefix_width = Const.SALT_WIDTH() + TSDB.metrics_width() + Const.TIMESTAMP_BYTES;
final FuzzyRowFilter fuzzy_filter;
if (explicit_tags && enable_fuzzy_filter && row_key_literals != null && !row_key_literals.isEmpty()) {
final byte[] fuzzy_key = new byte[prefix_width + (row_key_literals.size() * (TSDB.tagk_width() + TSDB.tagv_width()))];
System.arraycopy(scanner.getCurrentKey(), 0, fuzzy_key, 0, scanner.getCurrentKey().length);
final List<FuzzyFilterPair> fuzzy_filter_pairs = buildFuzzyFilters(row_key_literals, fuzzy_key);
// The Fuzzy Filter list is sorted: the first and last filters row key
// can be used to build the stop key for the scanner
final byte[] stop_key = Arrays.copyOf(fuzzy_filter_pairs.get(fuzzy_filter_pairs.size() - 1).getRowKey(), fuzzy_key.length);
System.arraycopy(scanner.getCurrentKey(), 0, stop_key, 0, prefix_width);
Internal.setBaseTime(stop_key, end_time);
int idx = prefix_width + TSDB.tagk_width();
// max out the tag values
while (idx < stop_key.length) {
for (int i = 0; i < TSDB.tagv_width(); i++) {
stop_key[idx++] = (byte) 0xFF;
}
idx += TSDB.tagk_width();
}
scanner.setStartKey(fuzzy_key);
scanner.setStopKey(stop_key);
fuzzy_filter = new FuzzyRowFilter(fuzzy_filter_pairs);
} else {
fuzzy_filter = null;
}
final String regex = getRowKeyUIDRegex(row_key_literals, explicit_tags);
final KeyRegexpFilter regex_filter;
if (!Strings.isNullOrEmpty(regex)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Regex for scanner: " + scanner + ": " + byteRegexToString(regex));
}
regex_filter = new KeyRegexpFilter(regex.toString(), Const.ASCII_CHARSET);
} else {
regex_filter = null;
}
if (fuzzy_filter != null && !Strings.isNullOrEmpty(regex)) {
final FilterList filter = new FilterList(Lists.newArrayList(fuzzy_filter, regex_filter), Operator.MUST_PASS_ALL);
scanner.setFilter(filter);
} else if (fuzzy_filter != null) {
scanner.setFilter(fuzzy_filter);
} else if (!Strings.isNullOrEmpty(regex)) {
scanner.setFilter(regex_filter);
}
}
Aggregations