Examples with FuzzyFilterPair org.hbase.async.FuzzyRowFilter.FuzzyFilterPair used on opensource projects

Example 1 with FuzzyFilterPair

use of org.hbase.async.FuzzyRowFilter.FuzzyFilterPair in project opentsdb by OpenTSDB.

the class QueryUtil method buildFuzzyFilters.

/**
 * Crafts a list of FuzzyFilters for scanning over data table rows and
 * filtering time series that the user doesn't want.
 * Note: The caller has to restrict the scan to proper start and stop
 * for the filter to work correctly.
 * @param row_key_literals A list of key value pairs to filter on.
 * @param fuzzy_key The starting row key we'll adjust for proper filtering.
 * @return A sorted, non-empty list of FuzzyFilterPair
 */
private static List<FuzzyFilterPair> buildFuzzyFilters(final ByteMap<byte[][]> row_key_literals, final byte[] fuzzy_key) {
    final int prefix_width = Const.SALT_WIDTH() + TSDB.metrics_width() + Const.TIMESTAMP_BYTES;
    final short name_width = TSDB.tagk_width();
    final short value_width = TSDB.tagv_width();
    final short tag_width = (short) (name_width + value_width);
    int row_key_size = prefix_width;
    if (row_key_literals != null) {
        for (byte[][] v : row_key_literals.values()) {
            final boolean not_key = v != null && v.length == 0;
            if (!not_key) {
                row_key_size += tag_width;
            }
        }
    }
    final List<FuzzyFilterPair> fuzzy_filter_pairs = new ArrayList<FuzzyFilterPair>(row_key_literals.size());
    // Initialize first_fuzzy_key and first_fuzzy_mask
    // these will serve as model for the fuzzy filter list
    // generated for tags with multiple values (|)
    byte[] first_fuzzy_key = Arrays.copyOf(fuzzy_key, fuzzy_key.length);
    byte[] first_fuzzy_mask = new byte[fuzzy_key.length];
    int fuzzy_offset = 0;
    // of the scanner)
    while (fuzzy_offset < prefix_width) {
        first_fuzzy_key[fuzzy_offset] = 0;
        first_fuzzy_mask[fuzzy_offset++] = (row_key_literals != null) ? (byte) 1 : (byte) 0;
    }
    // first pass to build the key and mask
    Iterator<Entry<byte[], byte[][]>> it = row_key_literals.iterator();
    while (it.hasNext()) {
        Entry<byte[], byte[][]> entry = it.next();
        final boolean not_key = entry.getValue() != null && entry.getValue().length == 0;
        if (!not_key) {
            final byte[] tag_key = entry.getKey();
            System.arraycopy(tag_key, 0, first_fuzzy_key, fuzzy_offset, name_width);
            for (int i = 0; i < name_width; i++) {
                first_fuzzy_mask[fuzzy_offset++] = 0;
            }
            final byte[] tag_value;
            if (entry.getValue() != null && entry.getValue().length > 0) {
                tag_value = entry.getValue()[0];
            } else {
                tag_value = null;
            }
            if (tag_value != null) {
                System.arraycopy(tag_value, 0, first_fuzzy_key, fuzzy_offset, value_width);
                for (int i = 0; i < value_width; i++) {
                    first_fuzzy_mask[fuzzy_offset++] = 0;
                }
            } else {
                // not filtered with fuzzy filter -> skip
                for (int i = 0; i < value_width; i++) {
                    first_fuzzy_key[fuzzy_offset] = 0;
                    first_fuzzy_mask[fuzzy_offset++] = 1;
                }
            }
        }
    }
    fuzzy_filter_pairs.add(new FuzzyFilterPair(first_fuzzy_key, first_fuzzy_mask));
    // generate filters for all combinations of tag values using the first key
    // as the template.
    fuzzy_offset = prefix_width;
    it = row_key_literals.iterator();
    while (it.hasNext()) {
        final Entry<byte[], byte[][]> entry = it.next();
        fuzzy_offset += name_width;
        // for each value
        if (entry.getValue() != null && entry.getValue().length > 1) {
            for (int i = 1; i < entry.getValue().length; i++) {
                final byte[] tag_value = entry.getValue()[i];
                byte[] local_fuzzy_key = Arrays.copyOf(first_fuzzy_key, row_key_size);
                System.arraycopy(tag_value, 0, local_fuzzy_key, fuzzy_offset, value_width);
                fuzzy_filter_pairs.add(new FuzzyFilterPair(local_fuzzy_key, first_fuzzy_mask));
            }
        }
        fuzzy_offset += value_width;
    }
    // Sort filters list over rowkey
    Collections.sort(fuzzy_filter_pairs, FUZZY_FILTER_CMP);
    return fuzzy_filter_pairs;
}

Example 2 with FuzzyFilterPair

use of org.hbase.async.FuzzyRowFilter.FuzzyFilterPair in project opentsdb by OpenTSDB.

the class QueryUtil method setDataTableScanFilter.

/**
 * Sets a filter or filter list on the scanner based on whether or not the
 * query had tags it needed to match.
 * NOTE: This method will sort the group bys.
 * @param scanner The scanner to modify.
 * @param group_bys An optional list of tag keys that we want to group on. May
 * be null.
 * @param row_key_literals An optional list of key value pairs to filter on.
 * May be null.
 * @param explicit_tags Whether or not explicit tags are enabled so that the
 * regex only picks out series with the specified tags
 * @param enable_fuzzy_filter Whether or not a fuzzy filter should be used
 * in combination with the explicit tags param. If explicit tags is disabled
 * then this param is ignored.
 * @param end_time The end of the query time so the fuzzy filter knows when
 * to stop scanning.
 */
public static void setDataTableScanFilter(final Scanner scanner, final List<byte[]> group_bys, final ByteMap<byte[][]> row_key_literals, final boolean explicit_tags, final boolean enable_fuzzy_filter, final int end_time) {
    // no-op
    if ((group_bys == null || group_bys.isEmpty()) && (row_key_literals == null || row_key_literals.isEmpty())) {
        return;
    }
    if (group_bys != null) {
        Collections.sort(group_bys, Bytes.MEMCMP);
    }
    final int prefix_width = Const.SALT_WIDTH() + TSDB.metrics_width() + Const.TIMESTAMP_BYTES;
    final FuzzyRowFilter fuzzy_filter;
    if (explicit_tags && enable_fuzzy_filter && row_key_literals != null && !row_key_literals.isEmpty()) {
        final byte[] fuzzy_key = new byte[prefix_width + (row_key_literals.size() * (TSDB.tagk_width() + TSDB.tagv_width()))];
        System.arraycopy(scanner.getCurrentKey(), 0, fuzzy_key, 0, scanner.getCurrentKey().length);
        final List<FuzzyFilterPair> fuzzy_filter_pairs = buildFuzzyFilters(row_key_literals, fuzzy_key);
        // The Fuzzy Filter list is sorted: the first and last filters row key
        // can be used to build the stop key for the scanner
        final byte[] stop_key = Arrays.copyOf(fuzzy_filter_pairs.get(fuzzy_filter_pairs.size() - 1).getRowKey(), fuzzy_key.length);
        System.arraycopy(scanner.getCurrentKey(), 0, stop_key, 0, prefix_width);
        Internal.setBaseTime(stop_key, end_time);
        int idx = prefix_width + TSDB.tagk_width();
        // max out the tag values
        while (idx < stop_key.length) {
            for (int i = 0; i < TSDB.tagv_width(); i++) {
                stop_key[idx++] = (byte) 0xFF;
            }
            idx += TSDB.tagk_width();
        }
        scanner.setStartKey(fuzzy_key);
        scanner.setStopKey(stop_key);
        fuzzy_filter = new FuzzyRowFilter(fuzzy_filter_pairs);
    } else {
        fuzzy_filter = null;
    }
    final String regex = getRowKeyUIDRegex(row_key_literals, explicit_tags);
    final KeyRegexpFilter regex_filter;
    if (!Strings.isNullOrEmpty(regex)) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Regex for scanner: " + scanner + ": " + byteRegexToString(regex));
        }
        regex_filter = new KeyRegexpFilter(regex.toString(), Const.ASCII_CHARSET);
    } else {
        regex_filter = null;
    }
    if (fuzzy_filter != null && !Strings.isNullOrEmpty(regex)) {
        final FilterList filter = new FilterList(Lists.newArrayList(fuzzy_filter, regex_filter), Operator.MUST_PASS_ALL);
        scanner.setFilter(filter);
    } else if (fuzzy_filter != null) {
        scanner.setFilter(fuzzy_filter);
    } else if (!Strings.isNullOrEmpty(regex)) {
        scanner.setFilter(regex_filter);
    }
}