Examples with Scanner org.hbase.async.Scanner used on opensource projects

Example 21 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class QueryUtil method getMetricScanner.

/**
   * Compiles an HBase scanner against the main data table
   * @param tsdb The TSDB with a configured HBaseClient
   * @param salt_bucket An optional salt bucket ID for salting the start/stop
   * keys.
   * @param metric The metric to scan for
   * @param start The start time stamp in seconds
   * @param stop The stop timestamp in seconds
   * @param table The table name to scan over
   * @param family The table family to scan over
   * @return A scanner ready for processing.
   */
public static Scanner getMetricScanner(final TSDB tsdb, final int salt_bucket, final byte[] metric, final int start, final int stop, final byte[] table, final byte[] family) {
    final short metric_width = TSDB.metrics_width();
    final int metric_salt_width = metric_width + Const.SALT_WIDTH();
    final byte[] start_row = new byte[metric_salt_width + Const.TIMESTAMP_BYTES];
    final byte[] end_row = new byte[metric_salt_width + Const.TIMESTAMP_BYTES];
    if (Const.SALT_WIDTH() > 0) {
        final byte[] salt = RowKey.getSaltBytes(salt_bucket);
        System.arraycopy(salt, 0, start_row, 0, Const.SALT_WIDTH());
        System.arraycopy(salt, 0, end_row, 0, Const.SALT_WIDTH());
    }
    Bytes.setInt(start_row, start, metric_salt_width);
    Bytes.setInt(end_row, stop, metric_salt_width);
    System.arraycopy(metric, 0, start_row, Const.SALT_WIDTH(), metric_width);
    System.arraycopy(metric, 0, end_row, Const.SALT_WIDTH(), metric_width);
    final Scanner scanner = tsdb.getClient().newScanner(table);
    scanner.setMaxNumRows(tsdb.getConfig().scanner_maxNumRows());
    scanner.setStartKey(start_row);
    scanner.setStopKey(end_row);
    scanner.setFamily(family);
    return scanner;
}

Example 22 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class TSUIDQuery method getLastWriteTimes.

/**
   * Fetches a list of TSUIDs given the metric and optional tag pairs. The query
   * format is similar to TsdbQuery but doesn't support grouping operators for 
   * tags. Only TSUIDs that had "ts_counter" qualifiers will be returned.
   * <p>
   * NOTE: If you called {@link #setQuery(String, Map)} successfully this will
   * immediately scan the meta table. But if you used the CTOR to set the
   * metric and tags it will attempt to resolve those and may return an exception.
   * @return A map of TSUIDs to the last timestamp (in milliseconds) when the
   * "ts_counter" was updated. Note that the timestamp will be the time stored
   * by HBase, not the actual timestamp of the data point. If nothing was
   * found, the map will be empty but not null.
   * @throws IllegalArgumentException if the metric was not set or the tag map
   * was null
   */
public Deferred<ByteMap<Long>> getLastWriteTimes() {
    class ResolutionCB implements Callback<Deferred<ByteMap<Long>>, Object> {

        @Override
        public Deferred<ByteMap<Long>> call(Object arg0) throws Exception {
            final Scanner scanner = getScanner();
            scanner.setQualifier(TSMeta.COUNTER_QUALIFIER());
            final Deferred<ByteMap<Long>> results = new Deferred<ByteMap<Long>>();
            final ByteMap<Long> tsuids = new ByteMap<Long>();
            final class ErrBack implements Callback<Object, Exception> {

                @Override
                public Object call(final Exception e) throws Exception {
                    results.callback(e);
                    return null;
                }

                @Override
                public String toString() {
                    return "Error callback";
                }
            }
            /**
         * Scanner callback that will call itself while iterating through the 
         * tsdb-meta table
         */
            final class ScannerCB implements Callback<Object, ArrayList<ArrayList<KeyValue>>> {

                /**
           * Starts the scanner and is called recursively to fetch the next set of
           * rows from the scanner.
           * @return The map of spans if loaded successfully, null if no data was
           * found
           */
                public Object scan() {
                    return scanner.nextRows().addCallback(this).addErrback(new ErrBack());
                }

                /**
           * Loops through each row of the scanner results and parses out data
           * points and optional meta data
           * @return null if no rows were found, otherwise the TreeMap with spans
           */
                @Override
                public Object call(final ArrayList<ArrayList<KeyValue>> rows) throws Exception {
                    try {
                        if (rows == null) {
                            results.callback(tsuids);
                            return null;
                        }
                        for (final ArrayList<KeyValue> row : rows) {
                            final byte[] tsuid = row.get(0).key();
                            tsuids.put(tsuid, row.get(0).timestamp());
                        }
                        return scan();
                    } catch (Exception e) {
                        results.callback(e);
                        return null;
                    }
                }
            }
            new ScannerCB().scan();
            return results;
        }

        @Override
        public String toString() {
            return "Last counter time callback";
        }
    }
    if (metric_uid == null) {
        return resolveMetric().addCallbackDeferring(new ResolutionCB());
    }
    try {
        return new ResolutionCB().call(null);
    } catch (Exception e) {
        return Deferred.fromError(e);
    }
}

Example 23 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class Fsck method runFullTable.

/**
   * Fetches the max metric ID and splits the data table up amongst threads on
   * a naive split. By default we execute cores * 2 threads but the user can
   * specify more or fewer.
   * @throws Exception If something goes pear shaped.
   */
public void runFullTable() throws Exception {
    LOG.info("Starting full table scan");
    final long start_time = System.currentTimeMillis() / 1000;
    final int workers = options.threads() > 0 ? options.threads() : Runtime.getRuntime().availableProcessors() * 2;
    final List<Scanner> scanners = CliUtils.getDataTableScanners(tsdb, workers);
    LOG.info("Spooling up [" + scanners.size() + "] worker threads");
    final List<Thread> threads = new ArrayList<Thread>(scanners.size());
    int i = 0;
    for (final Scanner scanner : scanners) {
        final FsckWorker worker = new FsckWorker(scanner, i++);
        worker.setName("Fsck #" + i);
        worker.start();
        threads.add(worker);
    }
    final Thread reporter = new ProgressReporter();
    reporter.start();
    for (final Thread thread : threads) {
        thread.join();
        LOG.info("Thread [" + thread + "] Finished");
    }
    reporter.interrupt();
    logResults();
    final long duration = (System.currentTimeMillis() / 1000) - start_time;
    LOG.info("Completed fsck in [" + duration + "] seconds");
}

Example 24 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class Fsck method runQueries.

/**
   * Scans the rows matching one or more standard queries. An aggregator is still
   * required though it's ignored.
   * @param queries The queries to execute
   * @throws Exception If something goes pear shaped.
   */
public void runQueries(final List<Query> queries) throws Exception {
    final long start_time = System.currentTimeMillis() / 1000;
    // TODO - threadify it. We *could* have hundreds of queries and we don't 
    // want to create that many threads. For now we'll just execute each one
    // serially
    final Thread reporter = new ProgressReporter();
    reporter.start();
    for (final Query query : queries) {
        final List<Scanner> scanners = Internal.getScanners(query);
        final List<Thread> threads = new ArrayList<Thread>(scanners.size());
        int i = 0;
        for (final Scanner scanner : scanners) {
            final FsckWorker worker = new FsckWorker(scanner, i++);
            worker.setName("Fsck #" + i);
            worker.start();
            threads.add(worker);
        }
        for (final Thread thread : threads) {
            thread.join();
            LOG.info("Thread [" + thread + "] Finished");
        }
    }
    reporter.interrupt();
    logResults();
    final long duration = (System.currentTimeMillis() / 1000) - start_time;
    LOG.info("Completed fsck in [" + duration + "] seconds");
}

Example 25 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class MetaPurge method purgeUIDMeta.

/**
   * Scans the entire UID table and removes any UIDMeta objects found.
   * @return The total number of columns deleted
   */
public Deferred<Long> purgeUIDMeta() {
    // a list to store all pending deletes so we don't exit before they've 
    // completed
    final ArrayList<Deferred<Object>> delete_calls = new ArrayList<Deferred<Object>>();
    final Deferred<Long> result = new Deferred<Long>();
    /**
     * Scanner callback that will recursively call itself and loop through the
     * rows of the UID table, issuing delete requests for all of the columns in
     * a row that match a meta qualifier.
     */
    final class MetaScanner implements Callback<Deferred<Long>, ArrayList<ArrayList<KeyValue>>> {

        final Scanner scanner;

        public MetaScanner() {
            scanner = getScanner(tsdb.uidTable());
        }

        /**
       * Fetches the next group of rows from the scanner and sets this class as
       * a callback
       * @return The total number of columns deleted after completion
       */
        public Deferred<Long> scan() {
            return scanner.nextRows().addCallbackDeferring(this);
        }

        @Override
        public Deferred<Long> call(ArrayList<ArrayList<KeyValue>> rows) throws Exception {
            if (rows == null) {
                result.callback(columns);
                return null;
            }
            for (final ArrayList<KeyValue> row : rows) {
                // one delete request per row. We'll almost always delete the whole
                // row, so preallocate some ram.
                ArrayList<byte[]> qualifiers = new ArrayList<byte[]>(row.size());
                for (KeyValue column : row) {
                    if (Bytes.equals(TSMeta.META_QUALIFIER(), column.qualifier())) {
                        qualifiers.add(column.qualifier());
                    } else if (Bytes.equals("metric_meta".getBytes(CHARSET), column.qualifier())) {
                        qualifiers.add(column.qualifier());
                    } else if (Bytes.equals("tagk_meta".getBytes(CHARSET), column.qualifier())) {
                        qualifiers.add(column.qualifier());
                    } else if (Bytes.equals("tagv_meta".getBytes(CHARSET), column.qualifier())) {
                        qualifiers.add(column.qualifier());
                    }
                }
                if (qualifiers.size() > 0) {
                    columns += qualifiers.size();
                    final DeleteRequest delete = new DeleteRequest(tsdb.uidTable(), row.get(0).key(), NAME_FAMILY, qualifiers.toArray(new byte[qualifiers.size()][]));
                    delete_calls.add(tsdb.getClient().delete(delete));
                }
            }
            /**
         * Buffer callback used to wait on all of the delete calls for the
         * last set of rows returned from the scanner so we don't fill up the
         * deferreds array and OOM out.
         */
            final class ContinueCB implements Callback<Deferred<Long>, ArrayList<Object>> {

                @Override
                public Deferred<Long> call(ArrayList<Object> deletes) throws Exception {
                    LOG.debug("[" + thread_id + "] Processed [" + deletes.size() + "] delete calls");
                    delete_calls.clear();
                    return scan();
                }
            }
            // fetch the next set of rows after waiting for current set of delete
            // requests to complete
            Deferred.group(delete_calls).addCallbackDeferring(new ContinueCB());
            return null;
        }
    }
    // start the scan
    new MetaScanner().scan();
    return result;
}