Search in sources :

Example 1 with KeyRegexpFilter

use of org.hbase.async.KeyRegexpFilter in project opentsdb by OpenTSDB.

the class TestTsdbQueryQueries method filterExplicitTagsGroupByOK.

@Test
public void filterExplicitTagsGroupByOK() throws Exception {
    tsdb.getConfig().overrideConfig("tsd.query.enable_fuzzy", "true");
    storeLongTimeSeriesSeconds(true, false);
    HashMap<String, String> tags = new HashMap<String, String>(1);
    tags.put("host", "*");
    query.setStartTime(1356998400);
    query.setEndTime(1357041600);
    query.setExplicitTags(true);
    query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false);
    final DataPoints[] dps = query.run();
    assertNotNull(dps);
    assertEquals("sys.cpu.user", dps[0].metricName());
    assertTrue(dps[0].getAggregatedTags().isEmpty());
    assertNull(dps[0].getAnnotations());
    assertEquals("web01", dps[0].getTags().get("host"));
    int value = 1;
    for (DataPoint dp : dps[0]) {
        assertEquals(value, dp.longValue());
        value++;
    }
    assertEquals(300, dps[0].aggregatedSize());
    // assert fuzzy
    for (final MockScanner scanner : storage.getScanners()) {
        assertTrue(scanner.getFilter() instanceof FilterList);
        FilterList filter_list = (FilterList) scanner.getFilter();
        assertEquals(2, filter_list.size());
        assertTrue(filter_list.filters().get(0) instanceof FuzzyRowFilter);
        assertTrue(filter_list.filters().get(1) instanceof KeyRegexpFilter);
    }
}
Also used : MockScanner(net.opentsdb.storage.MockBase.MockScanner) HashMap(java.util.HashMap) KeyRegexpFilter(org.hbase.async.KeyRegexpFilter) FilterList(org.hbase.async.FilterList) FuzzyRowFilter(org.hbase.async.FuzzyRowFilter) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 2 with KeyRegexpFilter

use of org.hbase.async.KeyRegexpFilter in project opentsdb by OpenTSDB.

the class TestTsdbQueryQueries method filterExplicitTagsMissing.

@Test
public void filterExplicitTagsMissing() throws Exception {
    tsdb.getConfig().overrideConfig("tsd.query.enable_fuzzy", "true");
    when(tag_names.getIdAsync("colo")).thenReturn(Deferred.fromResult(new byte[] { 0, 0, 0, 4 }));
    when(tag_values.getIdAsync("lga")).thenReturn(Deferred.fromResult(new byte[] { 0, 0, 0, 4 }));
    storeLongTimeSeriesSeconds(true, false);
    HashMap<String, String> tags = new HashMap<String, String>(1);
    tags.put("host", "web01");
    tags.put("colo", "lga");
    query.setStartTime(1356998400);
    query.setEndTime(1357041600);
    query.setExplicitTags(true);
    query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false);
    final DataPoints[] dps = query.run();
    assertNotNull(dps);
    assertEquals(0, dps.length);
    // assert fuzzy
    for (final MockScanner scanner : storage.getScanners()) {
        assertTrue(scanner.getFilter() instanceof FilterList);
        FilterList filter_list = (FilterList) scanner.getFilter();
        assertEquals(2, filter_list.size());
        assertTrue(filter_list.filters().get(0) instanceof FuzzyRowFilter);
        assertTrue(filter_list.filters().get(1) instanceof KeyRegexpFilter);
    }
}
Also used : MockScanner(net.opentsdb.storage.MockBase.MockScanner) HashMap(java.util.HashMap) KeyRegexpFilter(org.hbase.async.KeyRegexpFilter) FilterList(org.hbase.async.FilterList) FuzzyRowFilter(org.hbase.async.FuzzyRowFilter) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 3 with KeyRegexpFilter

use of org.hbase.async.KeyRegexpFilter in project opentsdb by OpenTSDB.

the class TestTsdbQueryQueries method filterExplicitTagsOK.

@Test
public void filterExplicitTagsOK() throws Exception {
    tsdb.getConfig().overrideConfig("tsd.query.enable_fuzzy", "true");
    storeLongTimeSeriesSeconds(true, false);
    HashMap<String, String> tags = new HashMap<String, String>(1);
    tags.put("host", "web01");
    query.setStartTime(1356998400);
    query.setEndTime(1357041600);
    query.setExplicitTags(true);
    query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false);
    final DataPoints[] dps = query.run();
    assertNotNull(dps);
    assertEquals("sys.cpu.user", dps[0].metricName());
    assertTrue(dps[0].getAggregatedTags().isEmpty());
    assertNull(dps[0].getAnnotations());
    assertEquals("web01", dps[0].getTags().get("host"));
    int value = 1;
    for (DataPoint dp : dps[0]) {
        assertEquals(value, dp.longValue());
        value++;
    }
    assertEquals(300, dps[0].aggregatedSize());
    // assert fuzzy
    for (final MockScanner scanner : storage.getScanners()) {
        assertTrue(scanner.getFilter() instanceof FilterList);
        FilterList filter_list = (FilterList) scanner.getFilter();
        assertEquals(2, filter_list.size());
        assertTrue(filter_list.filters().get(0) instanceof FuzzyRowFilter);
        assertTrue(filter_list.filters().get(1) instanceof KeyRegexpFilter);
    }
}
Also used : MockScanner(net.opentsdb.storage.MockBase.MockScanner) HashMap(java.util.HashMap) KeyRegexpFilter(org.hbase.async.KeyRegexpFilter) FilterList(org.hbase.async.FilterList) FuzzyRowFilter(org.hbase.async.FuzzyRowFilter) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 4 with KeyRegexpFilter

use of org.hbase.async.KeyRegexpFilter in project opentsdb by OpenTSDB.

the class QueryUtil method setDataTableScanFilter.

/**
 * Sets a filter or filter list on the scanner based on whether or not the
 * query had tags it needed to match.
 * NOTE: This method will sort the group bys.
 * @param scanner The scanner to modify.
 * @param group_bys An optional list of tag keys that we want to group on. May
 * be null.
 * @param row_key_literals An optional list of key value pairs to filter on.
 * May be null.
 * @param explicit_tags Whether or not explicit tags are enabled so that the
 * regex only picks out series with the specified tags
 * @param enable_fuzzy_filter Whether or not a fuzzy filter should be used
 * in combination with the explicit tags param. If explicit tags is disabled
 * then this param is ignored.
 * @param end_time The end of the query time so the fuzzy filter knows when
 * to stop scanning.
 */
public static void setDataTableScanFilter(final Scanner scanner, final List<byte[]> group_bys, final ByteMap<byte[][]> row_key_literals, final boolean explicit_tags, final boolean enable_fuzzy_filter, final int end_time) {
    // no-op
    if ((group_bys == null || group_bys.isEmpty()) && (row_key_literals == null || row_key_literals.isEmpty())) {
        return;
    }
    if (group_bys != null) {
        Collections.sort(group_bys, Bytes.MEMCMP);
    }
    final int prefix_width = Const.SALT_WIDTH() + TSDB.metrics_width() + Const.TIMESTAMP_BYTES;
    final FuzzyRowFilter fuzzy_filter;
    if (explicit_tags && enable_fuzzy_filter && row_key_literals != null && !row_key_literals.isEmpty()) {
        final byte[] fuzzy_key = new byte[prefix_width + (row_key_literals.size() * (TSDB.tagk_width() + TSDB.tagv_width()))];
        System.arraycopy(scanner.getCurrentKey(), 0, fuzzy_key, 0, scanner.getCurrentKey().length);
        final List<FuzzyFilterPair> fuzzy_filter_pairs = buildFuzzyFilters(row_key_literals, fuzzy_key);
        // The Fuzzy Filter list is sorted: the first and last filters row key
        // can be used to build the stop key for the scanner
        final byte[] stop_key = Arrays.copyOf(fuzzy_filter_pairs.get(fuzzy_filter_pairs.size() - 1).getRowKey(), fuzzy_key.length);
        System.arraycopy(scanner.getCurrentKey(), 0, stop_key, 0, prefix_width);
        Internal.setBaseTime(stop_key, end_time);
        int idx = prefix_width + TSDB.tagk_width();
        // max out the tag values
        while (idx < stop_key.length) {
            for (int i = 0; i < TSDB.tagv_width(); i++) {
                stop_key[idx++] = (byte) 0xFF;
            }
            idx += TSDB.tagk_width();
        }
        scanner.setStartKey(fuzzy_key);
        scanner.setStopKey(stop_key);
        fuzzy_filter = new FuzzyRowFilter(fuzzy_filter_pairs);
    } else {
        fuzzy_filter = null;
    }
    final String regex = getRowKeyUIDRegex(row_key_literals, explicit_tags);
    final KeyRegexpFilter regex_filter;
    if (!Strings.isNullOrEmpty(regex)) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Regex for scanner: " + scanner + ": " + byteRegexToString(regex));
        }
        regex_filter = new KeyRegexpFilter(regex.toString(), Const.ASCII_CHARSET);
    } else {
        regex_filter = null;
    }
    if (fuzzy_filter != null && !Strings.isNullOrEmpty(regex)) {
        final FilterList filter = new FilterList(Lists.newArrayList(fuzzy_filter, regex_filter), Operator.MUST_PASS_ALL);
        scanner.setFilter(filter);
    } else if (fuzzy_filter != null) {
        scanner.setFilter(fuzzy_filter);
    } else if (!Strings.isNullOrEmpty(regex)) {
        scanner.setFilter(regex_filter);
    }
}
Also used : FuzzyFilterPair(org.hbase.async.FuzzyRowFilter.FuzzyFilterPair) KeyRegexpFilter(org.hbase.async.KeyRegexpFilter) FilterList(org.hbase.async.FilterList) FuzzyRowFilter(org.hbase.async.FuzzyRowFilter)

Aggregations

FilterList (org.hbase.async.FilterList)4 FuzzyRowFilter (org.hbase.async.FuzzyRowFilter)4 KeyRegexpFilter (org.hbase.async.KeyRegexpFilter)4 HashMap (java.util.HashMap)3 MockScanner (net.opentsdb.storage.MockBase.MockScanner)3 Test (org.junit.Test)3 PrepareForTest (org.powermock.core.classloader.annotations.PrepareForTest)3 FuzzyFilterPair (org.hbase.async.FuzzyRowFilter.FuzzyFilterPair)1