Examples with FilterList org.hbase.async.FilterList used on opensource projects

Example 1 with FilterList

use of org.hbase.async.FilterList in project opentsdb by OpenTSDB.

the class TestTsdbQueryQueries method filterExplicitTagsGroupByOK.

@Test
public void filterExplicitTagsGroupByOK() throws Exception {
    tsdb.getConfig().overrideConfig("tsd.query.enable_fuzzy", "true");
    storeLongTimeSeriesSeconds(true, false);
    HashMap<String, String> tags = new HashMap<String, String>(1);
    tags.put("host", "*");
    query.setStartTime(1356998400);
    query.setEndTime(1357041600);
    query.setExplicitTags(true);
    query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false);
    final DataPoints[] dps = query.run();
    assertNotNull(dps);
    assertEquals("sys.cpu.user", dps[0].metricName());
    assertTrue(dps[0].getAggregatedTags().isEmpty());
    assertNull(dps[0].getAnnotations());
    assertEquals("web01", dps[0].getTags().get("host"));
    int value = 1;
    for (DataPoint dp : dps[0]) {
        assertEquals(value, dp.longValue());
        value++;
    }
    assertEquals(300, dps[0].aggregatedSize());
    // assert fuzzy
    for (final MockScanner scanner : storage.getScanners()) {
        assertTrue(scanner.getFilter() instanceof FilterList);
        FilterList filter_list = (FilterList) scanner.getFilter();
        assertEquals(2, filter_list.size());
        assertTrue(filter_list.filters().get(0) instanceof FuzzyRowFilter);
        assertTrue(filter_list.filters().get(1) instanceof KeyRegexpFilter);
    }
}

Example 2 with FilterList

use of org.hbase.async.FilterList in project opentsdb by OpenTSDB.

the class TsdbQuery method getScanner.

/**
 * Returns a scanner set for the given metric (from {@link #metric} or from
 * the first TSUID in the {@link #tsuids}s list. If one or more tags are
 * provided, it calls into {@link #createAndSetFilter} to setup a row key
 * filter. If one or more TSUIDs have been provided, it calls into
 * {@link #createAndSetTSUIDFilter} to setup a row key filter.
 * @param salt_bucket The salt bucket to scan over when salting is enabled.
 * @return A scanner to use for fetching data points
 */
protected Scanner getScanner(final int salt_bucket) throws HBaseException {
    final short metric_width = tsdb.metrics.width();
    // set the metric UID based on the TSUIDs if given, or the metric UID
    if (tsuids != null && !tsuids.isEmpty()) {
        final String tsuid = tsuids.get(0);
        final String metric_uid = tsuid.substring(0, metric_width * 2);
        metric = UniqueId.stringToUid(metric_uid);
    }
    final boolean is_rollup = RollupQuery.isValidQuery(rollup_query);
    // We search at least one row before and one row after the start & end
    // time we've been given as it's quite likely that the exact timestamp
    // we're looking for is in the middle of a row.  Plus, a number of things
    // rely on having a few extra data points before & after the exact start
    // & end dates in order to do proper rate calculation or downsampling near
    // the "edges" of the graph.
    final Scanner scanner = QueryUtil.getMetricScanner(tsdb, salt_bucket, metric, (int) getScanStartTimeSeconds(), end_time == UNSET ? // Will scan until the end (0xFFF...).
    -1 : (int) getScanEndTimeSeconds(), tableToBeScanned(), TSDB.FAMILY());
    if (tsdb.getConfig().use_otsdb_timestamp()) {
        long stTime = (getScanStartTimeSeconds() * 1000);
        long endTime = end_time == UNSET ? -1 : (getScanEndTimeSeconds() * 1000);
        if (tsdb.getConfig().get_date_tiered_compaction_start() <= stTime && rollup_query == null) {
            // TODO - we could set this for rollups but we also need to write
            // the rollup columns at the proper time.
            scanner.setTimeRange(stTime, endTime);
        }
    }
    if (tsuids != null && !tsuids.isEmpty()) {
        createAndSetTSUIDFilter(scanner);
    } else if (filters.size() > 0) {
        createAndSetFilter(scanner);
    }
    if (is_rollup) {
        ScanFilter existing = scanner.getFilter();
        // it. If not, then we can do this
        if (!rollup_query.getRollupAgg().toString().equals("avg")) {
            if (existing != null) {
                final List<ScanFilter> filters = new ArrayList<ScanFilter>(2);
                filters.add(existing);
                final List<ScanFilter> rollup_filters = new ArrayList<ScanFilter>(2);
                rollup_filters.add(new QualifierFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator(rollup_query.getRollupAgg().toString().getBytes(Const.ASCII_CHARSET))));
                rollup_filters.add(new QualifierFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator(new byte[] { (byte) tsdb.getRollupConfig().getIdForAggregator(rollup_query.getRollupAgg().toString()) })));
                filters.add(new FilterList(rollup_filters, Operator.MUST_PASS_ONE));
                scanner.setFilter(new FilterList(filters, Operator.MUST_PASS_ALL));
            } else {
                final List<ScanFilter> filters = new ArrayList<ScanFilter>(2);
                filters.add(new QualifierFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator(rollup_query.getRollupAgg().toString().getBytes(Const.ASCII_CHARSET))));
                filters.add(new QualifierFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator(new byte[] { (byte) tsdb.getRollupConfig().getIdForAggregator(rollup_query.getRollupAgg().toString()) })));
                scanner.setFilter(new FilterList(filters, Operator.MUST_PASS_ONE));
            }
        } else {
            final List<ScanFilter> filters = new ArrayList<ScanFilter>(4);
            filters.add(new QualifierFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator("sum".getBytes())));
            filters.add(new QualifierFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator("count".getBytes())));
            filters.add(new QualifierFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator(new byte[] { (byte) tsdb.getRollupConfig().getIdForAggregator("sum") })));
            filters.add(new QualifierFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator(new byte[] { (byte) tsdb.getRollupConfig().getIdForAggregator("count") })));
            if (existing != null) {
                final List<ScanFilter> combined = new ArrayList<ScanFilter>(2);
                combined.add(existing);
                combined.add(new FilterList(filters, Operator.MUST_PASS_ONE));
                scanner.setFilter(new FilterList(combined, Operator.MUST_PASS_ALL));
            } else {
                scanner.setFilter(new FilterList(filters, Operator.MUST_PASS_ONE));
            }
        }
    }
    return scanner;
}

Example 3 with FilterList

use of org.hbase.async.FilterList in project opentsdb by OpenTSDB.

the class TestTsdbQueryQueries method filterExplicitTagsMissing.

@Test
public void filterExplicitTagsMissing() throws Exception {
    tsdb.getConfig().overrideConfig("tsd.query.enable_fuzzy", "true");
    when(tag_names.getIdAsync("colo")).thenReturn(Deferred.fromResult(new byte[] { 0, 0, 0, 4 }));
    when(tag_values.getIdAsync("lga")).thenReturn(Deferred.fromResult(new byte[] { 0, 0, 0, 4 }));
    storeLongTimeSeriesSeconds(true, false);
    HashMap<String, String> tags = new HashMap<String, String>(1);
    tags.put("host", "web01");
    tags.put("colo", "lga");
    query.setStartTime(1356998400);
    query.setEndTime(1357041600);
    query.setExplicitTags(true);
    query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false);
    final DataPoints[] dps = query.run();
    assertNotNull(dps);
    assertEquals(0, dps.length);
    // assert fuzzy
    for (final MockScanner scanner : storage.getScanners()) {
        assertTrue(scanner.getFilter() instanceof FilterList);
        FilterList filter_list = (FilterList) scanner.getFilter();
        assertEquals(2, filter_list.size());
        assertTrue(filter_list.filters().get(0) instanceof FuzzyRowFilter);
        assertTrue(filter_list.filters().get(1) instanceof KeyRegexpFilter);
    }
}

Example 4 with FilterList

use of org.hbase.async.FilterList in project opentsdb by OpenTSDB.

the class TestTsdbQueryQueries method filterExplicitTagsOK.

@Test
public void filterExplicitTagsOK() throws Exception {
    tsdb.getConfig().overrideConfig("tsd.query.enable_fuzzy", "true");
    storeLongTimeSeriesSeconds(true, false);
    HashMap<String, String> tags = new HashMap<String, String>(1);
    tags.put("host", "web01");
    query.setStartTime(1356998400);
    query.setEndTime(1357041600);
    query.setExplicitTags(true);
    query.setTimeSeries("sys.cpu.user", tags, Aggregators.SUM, false);
    final DataPoints[] dps = query.run();
    assertNotNull(dps);
    assertEquals("sys.cpu.user", dps[0].metricName());
    assertTrue(dps[0].getAggregatedTags().isEmpty());
    assertNull(dps[0].getAnnotations());
    assertEquals("web01", dps[0].getTags().get("host"));
    int value = 1;
    for (DataPoint dp : dps[0]) {
        assertEquals(value, dp.longValue());
        value++;
    }
    assertEquals(300, dps[0].aggregatedSize());
    // assert fuzzy
    for (final MockScanner scanner : storage.getScanners()) {
        assertTrue(scanner.getFilter() instanceof FilterList);
        FilterList filter_list = (FilterList) scanner.getFilter();
        assertEquals(2, filter_list.size());
        assertTrue(filter_list.filters().get(0) instanceof FuzzyRowFilter);
        assertTrue(filter_list.filters().get(1) instanceof KeyRegexpFilter);
    }
}

Example 5 with FilterList

use of org.hbase.async.FilterList in project opentsdb by OpenTSDB.

the class QueryUtil method setDataTableScanFilter.

/**
 * Sets a filter or filter list on the scanner based on whether or not the
 * query had tags it needed to match.
 * NOTE: This method will sort the group bys.
 * @param scanner The scanner to modify.
 * @param group_bys An optional list of tag keys that we want to group on. May
 * be null.
 * @param row_key_literals An optional list of key value pairs to filter on.
 * May be null.
 * @param explicit_tags Whether or not explicit tags are enabled so that the
 * regex only picks out series with the specified tags
 * @param enable_fuzzy_filter Whether or not a fuzzy filter should be used
 * in combination with the explicit tags param. If explicit tags is disabled
 * then this param is ignored.
 * @param end_time The end of the query time so the fuzzy filter knows when
 * to stop scanning.
 */
public static void setDataTableScanFilter(final Scanner scanner, final List<byte[]> group_bys, final ByteMap<byte[][]> row_key_literals, final boolean explicit_tags, final boolean enable_fuzzy_filter, final int end_time) {
    // no-op
    if ((group_bys == null || group_bys.isEmpty()) && (row_key_literals == null || row_key_literals.isEmpty())) {
        return;
    }
    if (group_bys != null) {
        Collections.sort(group_bys, Bytes.MEMCMP);
    }
    final int prefix_width = Const.SALT_WIDTH() + TSDB.metrics_width() + Const.TIMESTAMP_BYTES;
    final FuzzyRowFilter fuzzy_filter;
    if (explicit_tags && enable_fuzzy_filter && row_key_literals != null && !row_key_literals.isEmpty()) {
        final byte[] fuzzy_key = new byte[prefix_width + (row_key_literals.size() * (TSDB.tagk_width() + TSDB.tagv_width()))];
        System.arraycopy(scanner.getCurrentKey(), 0, fuzzy_key, 0, scanner.getCurrentKey().length);
        final List<FuzzyFilterPair> fuzzy_filter_pairs = buildFuzzyFilters(row_key_literals, fuzzy_key);
        // The Fuzzy Filter list is sorted: the first and last filters row key
        // can be used to build the stop key for the scanner
        final byte[] stop_key = Arrays.copyOf(fuzzy_filter_pairs.get(fuzzy_filter_pairs.size() - 1).getRowKey(), fuzzy_key.length);
        System.arraycopy(scanner.getCurrentKey(), 0, stop_key, 0, prefix_width);
        Internal.setBaseTime(stop_key, end_time);
        int idx = prefix_width + TSDB.tagk_width();
        // max out the tag values
        while (idx < stop_key.length) {
            for (int i = 0; i < TSDB.tagv_width(); i++) {
                stop_key[idx++] = (byte) 0xFF;
            }
            idx += TSDB.tagk_width();
        }
        scanner.setStartKey(fuzzy_key);
        scanner.setStopKey(stop_key);
        fuzzy_filter = new FuzzyRowFilter(fuzzy_filter_pairs);
    } else {
        fuzzy_filter = null;
    }
    final String regex = getRowKeyUIDRegex(row_key_literals, explicit_tags);
    final KeyRegexpFilter regex_filter;
    if (!Strings.isNullOrEmpty(regex)) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Regex for scanner: " + scanner + ": " + byteRegexToString(regex));
        }
        regex_filter = new KeyRegexpFilter(regex.toString(), Const.ASCII_CHARSET);
    } else {
        regex_filter = null;
    }
    if (fuzzy_filter != null && !Strings.isNullOrEmpty(regex)) {
        final FilterList filter = new FilterList(Lists.newArrayList(fuzzy_filter, regex_filter), Operator.MUST_PASS_ALL);
        scanner.setFilter(filter);
    } else if (fuzzy_filter != null) {
        scanner.setFilter(fuzzy_filter);
    } else if (!Strings.isNullOrEmpty(regex)) {
        scanner.setFilter(regex_filter);
    }
}