Examples with Scanner org.hbase.async.Scanner used on opensource projects

Example 1 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class TestCliUtils method setupGetDataTableScanners.

private void setupGetDataTableScanners(final long max) {
    final KeyValue kv = new KeyValue(new byte[] {}, TSDB.FAMILY(), "metrics".getBytes(), Bytes.fromLong(max));
    final ArrayList<KeyValue> kvs = new ArrayList<KeyValue>(1);
    kvs.add(kv);
    when(client.get(any(GetRequest.class))).thenReturn(Deferred.<ArrayList<KeyValue>>fromResult(kvs));
    start_keys = new ArrayList<byte[]>();
    stop_keys = new ArrayList<byte[]>();
    final Scanner scanner = mock(Scanner.class);
    when(client.newScanner(any(byte[].class))).thenReturn(scanner);
    PowerMockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(final InvocationOnMock invocation) throws Throwable {
            start_keys.add((byte[]) invocation.getArguments()[0]);
            return null;
        }
    }).when(scanner).setStartKey(any(byte[].class));
    PowerMockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(final InvocationOnMock invocation) throws Throwable {
            stop_keys.add((byte[]) invocation.getArguments()[0]);
            return null;
        }
    }).when(scanner).setStopKey(any(byte[].class));
}

Example 2 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class TestUniqueId method suggestWithMatches.

@Test
public void suggestWithMatches() {
    uid = new UniqueId(client, table, METRIC, 3);
    final Scanner fake_scanner = mock(Scanner.class);
    when(client.newScanner(table)).thenReturn(fake_scanner);
    final ArrayList<ArrayList<KeyValue>> rows = new ArrayList<ArrayList<KeyValue>>(2);
    final byte[] foo_bar_id = { 0, 0, 1 };
    {
        ArrayList<KeyValue> row = new ArrayList<KeyValue>(1);
        row.add(new KeyValue("foo.bar".getBytes(), ID, METRIC_ARRAY, foo_bar_id));
        rows.add(row);
        row = new ArrayList<KeyValue>(1);
        row.add(new KeyValue("foo.baz".getBytes(), ID, METRIC_ARRAY, new byte[] { 0, 0, 2 }));
        rows.add(row);
    }
    when(fake_scanner.nextRows()).thenReturn(Deferred.<ArrayList<ArrayList<KeyValue>>>fromResult(rows)).thenReturn(Deferred.<ArrayList<ArrayList<KeyValue>>>fromResult(null));
    // Watch this! ______,^   I'm writing C++ in Java!
    final List<String> suggestions = uid.suggest("foo");
    final ArrayList<String> expected = new ArrayList<String>(2);
    expected.add("foo.bar");
    expected.add("foo.baz");
    assertEquals(expected, suggestions);
    // Verify that we cached the forward + backwards mapping for both results
    // we "discovered" as a result of the scan.
    assertEquals(4, uid.cacheSize());
    assertEquals(0, uid.cacheHits());
    // Verify that the cached results are usable.
    // Should be a cache hit ...
    assertArrayEquals(foo_bar_id, uid.getOrCreateId("foo.bar"));
    assertEquals(1, uid.cacheHits());
    // ... so verify there was no HBase Get.
    verify(client, never()).get(anyGet());
}

Example 3 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class SaltScanner method scan.

/**
   * Starts all of the scanners asynchronously and returns the data fetched
   * once all of the scanners have completed. Note that the result may be an
   * exception if one or more of the scanners encountered an exception. The 
   * first error will be returned, others will be logged. 
   * @return A deferred to wait on for results.
   */
public Deferred<TreeMap<byte[], Span>> scan() {
    start_time = System.currentTimeMillis();
    int i = 0;
    for (final Scanner scanner : scanners) {
        new ScannerCB(scanner, i++).scan();
    }
    return results;
}

Example 4 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class TsdbQuery method getScanner.

/**
   * Returns a scanner set for the given metric (from {@link #metric} or from
   * the first TSUID in the {@link #tsuids}s list. If one or more tags are 
   * provided, it calls into {@link #createAndSetFilter} to setup a row key 
   * filter. If one or more TSUIDs have been provided, it calls into
   * {@link #createAndSetTSUIDFilter} to setup a row key filter.
   * @param salt_bucket The salt bucket to scan over when salting is enabled.
   * @return A scanner to use for fetching data points
   */
protected Scanner getScanner(final int salt_bucket) throws HBaseException {
    final short metric_width = tsdb.metrics.width();
    // set the metric UID based on the TSUIDs if given, or the metric UID
    if (tsuids != null && !tsuids.isEmpty()) {
        final String tsuid = tsuids.get(0);
        final String metric_uid = tsuid.substring(0, metric_width * 2);
        metric = UniqueId.stringToUid(metric_uid);
    }
    // We search at least one row before and one row after the start & end
    // time we've been given as it's quite likely that the exact timestamp
    // we're looking for is in the middle of a row.  Plus, a number of things
    // rely on having a few extra data points before & after the exact start
    // & end dates in order to do proper rate calculation or downsampling near
    // the "edges" of the graph.
    final Scanner scanner = QueryUtil.getMetricScanner(tsdb, salt_bucket, metric, (int) getScanStartTimeSeconds(), end_time == UNSET ? // Will scan until the end (0xFFF...).
    -1 : (int) getScanEndTimeSeconds(), tsdb.table, TSDB.FAMILY());
    if (tsuids != null && !tsuids.isEmpty()) {
        createAndSetTSUIDFilter(scanner);
    } else if (filters.size() > 0) {
        createAndSetFilter(scanner);
    }
    return scanner;
}

Example 5 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class TsdbQuery method findSpans.

/**
   * Finds all the {@link Span}s that match this query.
   * This is what actually scans the HBase table and loads the data into
   * {@link Span}s.
   * @return A map from HBase row key to the {@link Span} for that row key.
   * Since a {@link Span} actually contains multiple HBase rows, the row key
   * stored in the map has its timestamp zero'ed out.
   * @throws HBaseException if there was a problem communicating with HBase to
   * perform the search.
   * @throws IllegalArgumentException if bad data was retrieved from HBase.
   */
private Deferred<TreeMap<byte[], Span>> findSpans() throws HBaseException {
    final short metric_width = tsdb.metrics.width();
    final // The key is a row key from HBase.
    TreeMap<byte[], Span> spans = new TreeMap<byte[], Span>(new SpanCmp((short) (Const.SALT_WIDTH() + metric_width)));
    // Copy only the filters that should trigger a tag resolution. If this list
    // is empty due to literals or a wildcard star, then we'll save a TON of
    // UID lookups
    final List<TagVFilter> scanner_filters;
    if (filters != null) {
        scanner_filters = new ArrayList<TagVFilter>(filters.size());
        for (final TagVFilter filter : filters) {
            if (filter.postScan()) {
                scanner_filters.add(filter);
            }
        }
    } else {
        scanner_filters = null;
    }
    if (Const.SALT_WIDTH() > 0) {
        final List<Scanner> scanners = new ArrayList<Scanner>(Const.SALT_BUCKETS());
        for (int i = 0; i < Const.SALT_BUCKETS(); i++) {
            scanners.add(getScanner(i));
        }
        scan_start_time = DateTime.nanoTime();
        return new SaltScanner(tsdb, metric, scanners, spans, scanner_filters, delete, query_stats, query_index).scan();
    }
    scan_start_time = DateTime.nanoTime();
    final Scanner scanner = getScanner();
    if (query_stats != null) {
        query_stats.addScannerId(query_index, 0, scanner.toString());
    }
    final Deferred<TreeMap<byte[], Span>> results = new Deferred<TreeMap<byte[], Span>>();
    /**
    * Scanner callback executed recursively each time we get a set of data
    * from storage. This is responsible for determining what columns are
    * returned and issuing requests to load leaf objects.
    * When the scanner returns a null set of rows, the method initiates the
    * final callback.
    */
    final class ScannerCB implements Callback<Object, ArrayList<ArrayList<KeyValue>>> {

        int nrows = 0;

        boolean seenAnnotation = false;

        long scanner_start = DateTime.nanoTime();

        long timeout = tsdb.getConfig().getLong("tsd.query.timeout");

        private final Set<String> skips = new HashSet<String>();

        private final Set<String> keepers = new HashSet<String>();

        // only used for salted scanners
        private final int index = 0;

        /** nanosecond timestamps */
        // reset each time we send an RPC to HBase
        private long fetch_start = 0;

        // cumulation of time waiting on HBase
        private long fetch_time = 0;

        // cumulation of time resolving UIDs
        private long uid_resolve_time = 0;

        private long uids_resolved = 0;

        // cumulation of time compacting
        private long compaction_time = 0;

        private long dps_pre_filter = 0;

        private long rows_pre_filter = 0;

        private long dps_post_filter = 0;

        private long rows_post_filter = 0;

        /** Error callback that will capture an exception from AsyncHBase and store
       * it so we can bubble it up to the caller.
       */
        class ErrorCB implements Callback<Object, Exception> {

            @Override
            public Object call(final Exception e) throws Exception {
                LOG.error("Scanner " + scanner + " threw an exception", e);
                close(e);
                return null;
            }
        }

        /**
      * Starts the scanner and is called recursively to fetch the next set of
      * rows from the scanner.
      * @return The map of spans if loaded successfully, null if no data was
      * found
      */
        public Object scan() {
            fetch_start = DateTime.nanoTime();
            return scanner.nextRows().addCallback(this).addErrback(new ErrorCB());
        }

        /**
      * Loops through each row of the scanner results and parses out data
      * points and optional meta data
      * @return null if no rows were found, otherwise the TreeMap with spans
      */
        @Override
        public Object call(final ArrayList<ArrayList<KeyValue>> rows) throws Exception {
            fetch_time += DateTime.nanoTime() - fetch_start;
            try {
                if (rows == null) {
                    scanlatency.add((int) DateTime.msFromNano(fetch_time));
                    LOG.info(TsdbQuery.this + " matched " + nrows + " rows in " + spans.size() + " spans in " + DateTime.msFromNano(fetch_time) + "ms");
                    close(null);
                    return null;
                }
                if (timeout > 0 && DateTime.msFromNanoDiff(DateTime.nanoTime(), scanner_start) > timeout) {
                    throw new InterruptedException("Query timeout exceeded!");
                }
                rows_pre_filter += rows.size();
                // used for UID resolution if a filter is involved
                final List<Deferred<Object>> lookups = filters != null && !filters.isEmpty() ? new ArrayList<Deferred<Object>>(rows.size()) : null;
                for (final ArrayList<KeyValue> row : rows) {
                    final byte[] key = row.get(0).key();
                    if (Bytes.memcmp(metric, key, 0, metric_width) != 0) {
                        scanner.close();
                        throw new IllegalDataException("HBase returned a row that doesn't match" + " our scanner (" + scanner + ")! " + row + " does not start" + " with " + Arrays.toString(metric));
                    }
                    // columns.
                    for (final KeyValue kv : row) {
                        if (kv.qualifier().length % 2 == 0) {
                            if (kv.qualifier().length == 2 || kv.qualifier().length == 4) {
                                ++dps_pre_filter;
                            } else {
                                // same precision. This is likely incorrect.
                                if (Internal.inMilliseconds(kv.qualifier())) {
                                    dps_pre_filter += (kv.qualifier().length / 4);
                                } else {
                                    dps_pre_filter += (kv.qualifier().length / 2);
                                }
                            }
                        } else if (kv.qualifier()[0] == AppendDataPoints.APPEND_COLUMN_PREFIX) {
                            // with appends we don't have a good rough estimate as the length
                            // can vary widely with the value length variability. Therefore we
                            // have to iterate.
                            int idx = 0;
                            int qlength = 0;
                            while (idx < kv.value().length) {
                                qlength = Internal.getQualifierLength(kv.value(), idx);
                                idx += qlength + Internal.getValueLengthFromQualifier(kv.value(), idx);
                                ++dps_pre_filter;
                            }
                        }
                    }
                    // TODO - byte set instead of a string for the uid may be faster
                    if (scanner_filters != null && !scanner_filters.isEmpty()) {
                        lookups.clear();
                        final String tsuid = UniqueId.uidToString(UniqueId.getTSUIDFromKey(key, TSDB.metrics_width(), Const.TIMESTAMP_BYTES));
                        if (skips.contains(tsuid)) {
                            continue;
                        }
                        if (!keepers.contains(tsuid)) {
                            final long uid_start = DateTime.nanoTime();
                            /** CB to called after all of the UIDs have been resolved */
                            class MatchCB implements Callback<Object, ArrayList<Boolean>> {

                                @Override
                                public Object call(final ArrayList<Boolean> matches) throws Exception {
                                    for (final boolean matched : matches) {
                                        if (!matched) {
                                            skips.add(tsuid);
                                            return null;
                                        }
                                    }
                                    // matched all, good data
                                    keepers.add(tsuid);
                                    processRow(key, row);
                                    return null;
                                }
                            }
                            /** Resolves all of the row key UIDs to their strings for filtering */
                            class GetTagsCB implements Callback<Deferred<ArrayList<Boolean>>, Map<String, String>> {

                                @Override
                                public Deferred<ArrayList<Boolean>> call(final Map<String, String> tags) throws Exception {
                                    uid_resolve_time += (DateTime.nanoTime() - uid_start);
                                    uids_resolved += tags.size();
                                    final List<Deferred<Boolean>> matches = new ArrayList<Deferred<Boolean>>(scanner_filters.size());
                                    for (final TagVFilter filter : scanner_filters) {
                                        matches.add(filter.match(tags));
                                    }
                                    return Deferred.group(matches);
                                }
                            }
                            lookups.add(Tags.getTagsAsync(tsdb, key).addCallbackDeferring(new GetTagsCB()).addBoth(new MatchCB()));
                        } else {
                            processRow(key, row);
                        }
                    } else {
                        processRow(key, row);
                    }
                }
                // if we don't have filters.
                if (lookups != null && lookups.size() > 0) {
                    class GroupCB implements Callback<Object, ArrayList<Object>> {

                        @Override
                        public Object call(final ArrayList<Object> group) throws Exception {
                            return scan();
                        }
                    }
                    return Deferred.group(lookups).addCallback(new GroupCB());
                } else {
                    return scan();
                }
            } catch (Exception e) {
                close(e);
                return null;
            }
        }

        /**
        * Finds or creates the span for this row, compacts it and stores it.
        * @param key The row key to use for fetching the span
        * @param row The row to add
        */
        void processRow(final byte[] key, final ArrayList<KeyValue> row) {
            ++rows_post_filter;
            if (delete) {
                final DeleteRequest del = new DeleteRequest(tsdb.dataTable(), key);
                tsdb.getClient().delete(del);
            }
            // columns.
            for (final KeyValue kv : row) {
                if (kv.qualifier().length % 2 == 0) {
                    if (kv.qualifier().length == 2 || kv.qualifier().length == 4) {
                        ++dps_post_filter;
                    } else {
                        // same precision. This is likely incorrect.
                        if (Internal.inMilliseconds(kv.qualifier())) {
                            dps_post_filter += (kv.qualifier().length / 4);
                        } else {
                            dps_post_filter += (kv.qualifier().length / 2);
                        }
                    }
                } else if (kv.qualifier()[0] == AppendDataPoints.APPEND_COLUMN_PREFIX) {
                    // with appends we don't have a good rough estimate as the length
                    // can vary widely with the value length variability. Therefore we
                    // have to iterate.
                    int idx = 0;
                    int qlength = 0;
                    while (idx < kv.value().length) {
                        qlength = Internal.getQualifierLength(kv.value(), idx);
                        idx += qlength + Internal.getValueLengthFromQualifier(kv.value(), idx);
                        ++dps_post_filter;
                    }
                }
            }
            Span datapoints = spans.get(key);
            if (datapoints == null) {
                datapoints = new Span(tsdb);
                spans.put(key, datapoints);
            }
            final long compaction_start = DateTime.nanoTime();
            final KeyValue compacted = tsdb.compact(row, datapoints.getAnnotations());
            compaction_time += (DateTime.nanoTime() - compaction_start);
            seenAnnotation |= !datapoints.getAnnotations().isEmpty();
            if (compacted != null) {
                // Can be null if we ignored all KVs.
                datapoints.addRow(compacted);
                ++nrows;
            }
        }

        void close(final Exception e) {
            scanner.close();
            if (query_stats != null) {
                query_stats.addScannerStat(query_index, index, QueryStat.SCANNER_TIME, DateTime.nanoTime() - scan_start_time);
                // Scanner Stats
                /* Uncomment when AsyncHBase has this feature:
           query_stats.addScannerStat(query_index, index, 
               QueryStat.ROWS_FROM_STORAGE, scanner.getRowsFetched());
           query_stats.addScannerStat(query_index, index, 
               QueryStat.COLUMNS_FROM_STORAGE, scanner.getColumnsFetched());
           query_stats.addScannerStat(query_index, index, 
               QueryStat.BYTES_FROM_STORAGE, scanner.getBytesFetched()); */
                query_stats.addScannerStat(query_index, index, QueryStat.HBASE_TIME, fetch_time);
                query_stats.addScannerStat(query_index, index, QueryStat.SUCCESSFUL_SCAN, e == null ? 1 : 0);
                // Post Scan stats
                query_stats.addScannerStat(query_index, index, QueryStat.ROWS_PRE_FILTER, rows_pre_filter);
                query_stats.addScannerStat(query_index, index, QueryStat.DPS_PRE_FILTER, dps_pre_filter);
                query_stats.addScannerStat(query_index, index, QueryStat.ROWS_POST_FILTER, rows_post_filter);
                query_stats.addScannerStat(query_index, index, QueryStat.DPS_POST_FILTER, dps_post_filter);
                query_stats.addScannerStat(query_index, index, QueryStat.SCANNER_UID_TO_STRING_TIME, uid_resolve_time);
                query_stats.addScannerStat(query_index, index, QueryStat.UID_PAIRS_RESOLVED, uids_resolved);
                query_stats.addScannerStat(query_index, index, QueryStat.COMPACTION_TIME, compaction_time);
            }
            if (e != null) {
                results.callback(e);
            } else if (nrows < 1 && !seenAnnotation) {
                results.callback(null);
            } else {
                results.callback(spans);
            }
        }
    }
    new ScannerCB().scan();
    return results;
}