Examples with Scanner org.hbase.async.Scanner used on opensource projects

Example 11 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class TSUIDQuery method getScanner.

/**
   * Configures the scanner for a specific metric and optional tags
   * @return A configured scanner
   */
private Scanner getScanner() {
    final Scanner scanner = tsdb.getClient().newScanner(tsdb.metaTable());
    scanner.setStartKey(metric_uid);
    // increment the metric UID by one so we can scan all of the rows for the
    // given metric
    final long stop = UniqueId.uidToLong(metric_uid, TSDB.metrics_width()) + 1;
    scanner.setStopKey(UniqueId.longToUID(stop, TSDB.metrics_width()));
    scanner.setFamily(TSMeta.FAMILY());
    // set the filter if we have tags
    if (!tags.isEmpty()) {
        final short name_width = TSDB.tagk_width();
        final short value_width = TSDB.tagv_width();
        final short tagsize = (short) (name_width + value_width);
        // Generate a regexp for our tags.  Say we have 2 tags: { 0 0 1 0 0 2 }
        // and { 4 5 6 9 8 7 }, the regexp will be:
        // "^.{7}(?:.{6})*\\Q\000\000\001\000\000\002\\E(?:.{6})*\\Q\004\005\006\011\010\007\\E(?:.{6})*$"
        final StringBuilder buf = new StringBuilder(// "^.{N}" + "(?:.{M})*" + "$"
        15 + (// "(?:.{M})*\\Q" + tagsize bytes + "\\E"
        (13 + tagsize) * (tags.size())));
        // Alright, let's build this regexp.  From the beginning...
        buf.append(// Ensure we use the DOTALL flag.
        "(?s)" + "^.{").append(TSDB.metrics_width()).append("}");
        final Iterator<byte[]> tags = this.tag_uids.iterator();
        byte[] tag = tags.hasNext() ? tags.next() : null;
        // regexp in order by ID, which means we just merge two sorted lists.
        do {
            // Skip any number of tags.
            buf.append("(?:.{").append(tagsize).append("})*\\Q");
            UniqueId.addIdToRegexp(buf, tag);
            tag = tags.hasNext() ? tags.next() : null;
        } while (// Stop when they both become null.
        tag != null);
        // Skip any number of tags before the end.
        buf.append("(?:.{").append(tagsize).append("})*$");
        scanner.setKeyRegexp(buf.toString(), CHARSET);
    }
    return scanner;
}

Example 12 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class TSUIDQuery method getTSMetas.

/**
   * Returns all TSMeta objects stored for timeseries defined by this query. The 
   * query is similar to TsdbQuery without any aggregations. Returns an empty 
   * list, when no TSMetas are found. Only returns stored TSMetas.
   * <p>
   * NOTE: If you called {@link #setQuery(String, Map)} successfully this will
   * immediately scan the meta table. But if you used the CTOR to set the
   * metric and tags it will attempt to resolve those and may return an exception.
   * @return A list of existing TSMetas for the timeseries covered by the query.
   * @throws IllegalArgumentException When either no metric was specified or the
   * tag map was null (Empty map is OK).
   */
public Deferred<List<TSMeta>> getTSMetas() {
    class ResolutionCB implements Callback<Deferred<List<TSMeta>>, Object> {

        @Override
        public Deferred<List<TSMeta>> call(final Object done) throws Exception {
            final Scanner scanner = getScanner();
            scanner.setQualifier(TSMeta.META_QUALIFIER());
            final Deferred<List<TSMeta>> results = new Deferred<List<TSMeta>>();
            final List<TSMeta> tsmetas = new ArrayList<TSMeta>();
            final List<Deferred<TSMeta>> tsmeta_group = new ArrayList<Deferred<TSMeta>>();
            final class TSMetaGroupCB implements Callback<Object, ArrayList<TSMeta>> {

                @Override
                public List<TSMeta> call(ArrayList<TSMeta> ts) throws Exception {
                    for (TSMeta tsm : ts) {
                        if (tsm != null) {
                            tsmetas.add(tsm);
                        }
                    }
                    results.callback(tsmetas);
                    return null;
                }

                @Override
                public String toString() {
                    return "TSMeta callback";
                }
            }
            final class ErrBack implements Callback<Object, Exception> {

                @Override
                public Object call(final Exception e) throws Exception {
                    results.callback(e);
                    return null;
                }

                @Override
                public String toString() {
                    return "Error callback";
                }
            }
            /**
         * Scanner callback that will call itself while iterating through the 
         * tsdb-meta table.
         * 
         * Keeps track of a Set of Deferred TSMeta calls. When all rows are scanned,
         * will wait for all TSMeta calls to be completed and then create the result 
         * list.
         */
            final class ScannerCB implements Callback<Object, ArrayList<ArrayList<KeyValue>>> {

                /**
           * Starts the scanner and is called recursively to fetch the next set of
           * rows from the scanner.
           * @return The map of spans if loaded successfully, null if no data was
           * found
           */
                public Object scan() {
                    return scanner.nextRows().addCallback(this).addErrback(new ErrBack());
                }

                /**
           * Loops through each row of the scanner results and parses out data
           * points and optional meta data
           * @return null if no rows were found, otherwise the TreeMap with spans
           */
                @Override
                public Object call(final ArrayList<ArrayList<KeyValue>> rows) throws Exception {
                    try {
                        if (rows == null) {
                            Deferred.group(tsmeta_group).addCallback(new TSMetaGroupCB()).addErrback(new ErrBack());
                            return null;
                        }
                        for (final ArrayList<KeyValue> row : rows) {
                            tsmeta_group.add(TSMeta.parseFromColumn(tsdb, row.get(0), true));
                        }
                        return scan();
                    } catch (Exception e) {
                        results.callback(e);
                        return null;
                    }
                }
            }
            new ScannerCB().scan();
            return results;
        }

        @Override
        public String toString() {
            return "TSMeta scan callback";
        }
    }
    if (metric_uid == null) {
        return resolveMetric().addCallbackDeferring(new ResolutionCB());
    }
    try {
        return new ResolutionCB().call(null);
    } catch (Exception e) {
        return Deferred.fromError(e);
    }
}

Example 13 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class MetaPurge method getScanner.

/**
   * Returns a scanner to run over the UID table starting at the given row
   * @return A scanner configured for the entire table
   * @throws HBaseException if something goes boom
   */
private Scanner getScanner(final byte[] table) throws HBaseException {
    short metric_width = TSDB.metrics_width();
    final byte[] start_row = Arrays.copyOfRange(Bytes.fromLong(start_id), 8 - metric_width, 8);
    final byte[] end_row = Arrays.copyOfRange(Bytes.fromLong(end_id), 8 - metric_width, 8);
    final Scanner scanner = tsdb.getClient().newScanner(table);
    scanner.setStartKey(start_row);
    scanner.setStopKey(end_row);
    scanner.setFamily(NAME_FAMILY);
    return scanner;
}

Example 14 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class TimeSeriesLookup method lookupAsync.

/**
   * Lookup time series associated with the given metric, tagk, tagv or tag 
   * pairs. Either the meta table or the data table will be scanned. If no
   * metric is given, a full table scan must be performed and this call may take
   * a long time to complete. 
   * When dumping to stdout, if an ID can't be looked up, it will be logged and
   * skipped.
   * @return A list of TSUIDs matching the given lookup query.
   * @throws NoSuchUniqueName if any of the given names fail to resolve to a 
   * UID.
   * @since 2.2
   */
public Deferred<List<byte[]>> lookupAsync() {
    final Pattern tagv_regex = tagv_filter != null ? Pattern.compile(tagv_filter) : null;
    // we don't really know what size the UIDs will resolve to so just grab
    // a decent amount.
    final StringBuffer buf = to_stdout ? new StringBuffer(2048) : null;
    final long start = System.currentTimeMillis();
    final int limit;
    if (query.getLimit() > 0) {
        if (query.useMeta() || Const.SALT_WIDTH() < 1) {
            limit = query.getLimit();
        } else if (query.getLimit() < Const.SALT_BUCKETS()) {
            limit = 1;
        } else {
            limit = query.getLimit() / Const.SALT_BUCKETS();
        }
    } else {
        limit = 0;
    }
    class ScannerCB implements Callback<Deferred<List<byte[]>>, ArrayList<ArrayList<KeyValue>>> {

        private final Scanner scanner;

        // used to avoid dupes when scanning the data table
        private byte[] last_tsuid = null;

        private int rows_read;

        ScannerCB(final Scanner scanner) {
            this.scanner = scanner;
        }

        Deferred<List<byte[]>> scan() {
            return scanner.nextRows().addCallbackDeferring(this);
        }

        @Override
        public Deferred<List<byte[]>> call(final ArrayList<ArrayList<KeyValue>> rows) throws Exception {
            if (rows == null) {
                scanner.close();
                if (query.useMeta() || Const.SALT_WIDTH() < 1) {
                    LOG.debug("Lookup query matched " + tsuids.size() + " time series in " + (System.currentTimeMillis() - start) + " ms");
                }
                return Deferred.fromResult(tsuids);
            }
            for (final ArrayList<KeyValue> row : rows) {
                if (limit > 0 && rows_read >= limit) {
                    // little recursion to close the scanner and log above.
                    return call(null);
                }
                final byte[] tsuid = query.useMeta() ? row.get(0).key() : UniqueId.getTSUIDFromKey(row.get(0).key(), TSDB.metrics_width(), Const.TIMESTAMP_BYTES);
                // string objects.
                if (tagv_regex != null && !tagv_regex.matcher(new String(tsuid, CHARSET)).find()) {
                    continue;
                }
                if (to_stdout) {
                    if (last_tsuid != null && Bytes.memcmp(last_tsuid, tsuid) == 0) {
                        continue;
                    }
                    last_tsuid = tsuid;
                    try {
                        buf.append(UniqueId.uidToString(tsuid)).append(" ");
                        buf.append(RowKey.metricNameAsync(tsdb, tsuid).joinUninterruptibly());
                        buf.append(" ");
                        final List<byte[]> tag_ids = UniqueId.getTagPairsFromTSUID(tsuid);
                        final Map<String, String> resolved_tags = Tags.resolveIdsAsync(tsdb, tag_ids).joinUninterruptibly();
                        for (final Map.Entry<String, String> tag_pair : resolved_tags.entrySet()) {
                            buf.append(tag_pair.getKey()).append("=").append(tag_pair.getValue()).append(" ");
                        }
                    } catch (NoSuchUniqueId nsui) {
                        LOG.error("Unable to resolve UID in TSUID (" + UniqueId.uidToString(tsuid) + ") " + nsui.getMessage());
                    }
                    // reset the buffer so we can re-use it
                    buf.setLength(0);
                } else {
                    tsuids.add(tsuid);
                }
                ++rows_read;
            }
            return scan();
        }

        @Override
        public String toString() {
            return "Scanner callback";
        }
    }
    class CompleteCB implements Callback<List<byte[]>, ArrayList<List<byte[]>>> {

        @Override
        public List<byte[]> call(final ArrayList<List<byte[]>> unused) throws Exception {
            LOG.debug("Lookup query matched " + tsuids.size() + " time series in " + (System.currentTimeMillis() - start) + " ms");
            return tsuids;
        }

        @Override
        public String toString() {
            return "Final async lookup callback";
        }
    }
    class UIDCB implements Callback<Deferred<List<byte[]>>, Object> {

        @Override
        public Deferred<List<byte[]>> call(Object arg0) throws Exception {
            if (!query.useMeta() && Const.SALT_WIDTH() > 0 && metric_uid != null) {
                final ArrayList<Deferred<List<byte[]>>> deferreds = new ArrayList<Deferred<List<byte[]>>>(Const.SALT_BUCKETS());
                for (int i = 0; i < Const.SALT_BUCKETS(); i++) {
                    deferreds.add(new ScannerCB(getScanner(i)).scan());
                }
                return Deferred.group(deferreds).addCallback(new CompleteCB());
            } else {
                return new ScannerCB(getScanner(0)).scan();
            }
        }

        @Override
        public String toString() {
            return "UID resolution callback";
        }
    }
    return resolveUIDs().addCallbackDeferring(new UIDCB());
}

Example 15 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class CliUtils method getDataTableScanner.

/**
   * Returns a scanner set to iterate over a range of metrics in the main 
   * tsdb-data table.
   * @param tsdb The TSDB to use for data access
   * @param start_id A metric ID to start scanning on
   * @param end_id A metric ID to end scanning on
   * @return A scanner on the "t" CF configured for the specified range
   * @throws HBaseException if something goes pear shaped
   */
static final Scanner getDataTableScanner(final TSDB tsdb, final long start_id, final long end_id) throws HBaseException {
    final short metric_width = TSDB.metrics_width();
    final byte[] start_row = Arrays.copyOfRange(Bytes.fromLong(start_id), 8 - metric_width, 8);
    final byte[] end_row = Arrays.copyOfRange(Bytes.fromLong(end_id), 8 - metric_width, 8);
    final Scanner scanner = tsdb.getClient().newScanner(tsdb.dataTable());
    scanner.setStartKey(start_row);
    scanner.setStopKey(end_row);
    scanner.setFamily(TSDB.FAMILY());
    return scanner;
}