Examples with Scanner org.hbase.async.Scanner used on opensource projects

Example 16 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class CliUtils method getDataTableScanners.

/**
   * Generates a list of Scanners to use for iterating over the full TSDB 
   * data table. If salting is enabled then {@link Const.SaltBukets()} scanners
   * will be returned. If salting is disabled then {@link num_scanners} 
   * scanners will be returned.
   * @param tsdb The TSDB to generate scanners from
   * @param num_scanners The max number of scanners if salting is disabled
   * @return A list of scanners to use for scanning the table.
   */
static final List<Scanner> getDataTableScanners(final TSDB tsdb, final int num_scanners) {
    if (num_scanners < 1) {
        throw new IllegalArgumentException("Number of scanners must be 1 or more: " + num_scanners);
    }
    // TODO - It would be neater to get a list of regions then create scanners
    // on those boundaries. We'll have to modify AsyncHBase for that to avoid
    // creating lots of custom HBase logic in here.
    final short metric_width = TSDB.metrics_width();
    final List<Scanner> scanners = new ArrayList<Scanner>();
    if (Const.SALT_WIDTH() > 0) {
        // salting is enabled so we'll create one scanner per salt for now
        byte[] start_key = HBaseClient.EMPTY_ARRAY;
        byte[] stop_key = HBaseClient.EMPTY_ARRAY;
        for (int i = 1; i < Const.SALT_BUCKETS() + 1; i++) {
            // move stop key to start key
            if (i > 1) {
                start_key = Arrays.copyOf(stop_key, stop_key.length);
            }
            if (i >= Const.SALT_BUCKETS()) {
                stop_key = HBaseClient.EMPTY_ARRAY;
            } else {
                stop_key = RowKey.getSaltBytes(i);
            }
            final Scanner scanner = tsdb.getClient().newScanner(tsdb.dataTable());
            scanner.setStartKey(Arrays.copyOf(start_key, start_key.length));
            scanner.setStopKey(Arrays.copyOf(stop_key, stop_key.length));
            scanner.setFamily(TSDB.FAMILY());
            scanners.add(scanner);
        }
    } else {
        // No salt, just go by the max metric ID
        long max_id = CliUtils.getMaxMetricID(tsdb);
        if (max_id < 1) {
            max_id = Internal.getMaxUnsignedValueOnBytes(metric_width);
        }
        final long quotient = max_id % num_scanners == 0 ? max_id / num_scanners : (max_id / num_scanners) + 1;
        byte[] start_key = HBaseClient.EMPTY_ARRAY;
        byte[] stop_key = new byte[metric_width];
        for (int i = 0; i < num_scanners; i++) {
            // move stop key to start key
            if (i > 0) {
                start_key = Arrays.copyOf(stop_key, stop_key.length);
            }
            // setup the next stop key
            final byte[] stop_id;
            if ((i + 1) * quotient > max_id) {
                stop_id = null;
            } else {
                stop_id = Bytes.fromLong((i + 1) * quotient);
            }
            if ((i + 1) * quotient >= max_id) {
                stop_key = HBaseClient.EMPTY_ARRAY;
            } else {
                System.arraycopy(stop_id, stop_id.length - metric_width, stop_key, 0, metric_width);
            }
            final Scanner scanner = tsdb.getClient().newScanner(tsdb.dataTable());
            scanner.setStartKey(Arrays.copyOf(start_key, start_key.length));
            if (stop_key != null) {
                scanner.setStopKey(Arrays.copyOf(stop_key, stop_key.length));
            }
            scanner.setFamily(TSDB.FAMILY());
            scanners.add(scanner);
        }
    }
    return scanners;
}

Example 17 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class Tree method fetchAllTrees.

/**
   * Attempts to retrieve all trees from the UID table, including their rules.
   * If no trees were found, the result will be an empty list
   * @param tsdb The TSDB to use for storage
   * @return A list of tree objects. May be empty if none were found
   */
public static Deferred<List<Tree>> fetchAllTrees(final TSDB tsdb) {
    final Deferred<List<Tree>> result = new Deferred<List<Tree>>();
    /**
     * Scanner callback that recursively calls itself to load the next set of
     * rows from storage. When the scanner returns a null, the callback will
     * return with the list of trees discovered.
     */
    final class AllTreeScanner implements Callback<Object, ArrayList<ArrayList<KeyValue>>> {

        private final List<Tree> trees = new ArrayList<Tree>();

        private final Scanner scanner;

        public AllTreeScanner() {
            scanner = setupAllTreeScanner(tsdb);
        }

        /**
       * Fetches the next set of results from the scanner and adds this class
       * as a callback.
       * @return A list of trees if the scanner has reached the end
       */
        public Object fetchTrees() {
            return scanner.nextRows().addCallback(this);
        }

        @Override
        public Object call(ArrayList<ArrayList<KeyValue>> rows) throws Exception {
            if (rows == null) {
                result.callback(trees);
                return null;
            }
            for (ArrayList<KeyValue> row : rows) {
                final Tree tree = new Tree();
                for (KeyValue column : row) {
                    if (column.qualifier().length >= TREE_QUALIFIER.length && Bytes.memcmp(TREE_QUALIFIER, column.qualifier()) == 0) {
                        // it's *this* tree. We deserialize to a new object and copy
                        // since the columns could be in any order and we may get a rule 
                        // before the tree object
                        final Tree local_tree = JSON.parseToObject(column.value(), Tree.class);
                        tree.created = local_tree.created;
                        tree.description = local_tree.description;
                        tree.name = local_tree.name;
                        tree.notes = local_tree.notes;
                        tree.strict_match = local_tree.strict_match;
                        tree.enabled = local_tree.enabled;
                        tree.store_failures = local_tree.store_failures;
                        // WARNING: Since the JSON data in storage doesn't contain the tree
                        // ID, we need to parse it from the row key
                        tree.setTreeId(bytesToId(row.get(0).key()));
                    // tree rule
                    } else if (column.qualifier().length > TreeRule.RULE_PREFIX().length && Bytes.memcmp(TreeRule.RULE_PREFIX(), column.qualifier(), 0, TreeRule.RULE_PREFIX().length) == 0) {
                        final TreeRule rule = TreeRule.parseFromStorage(column);
                        tree.addRule(rule);
                    }
                }
                // only add the tree if we parsed a valid ID
                if (tree.tree_id > 0) {
                    trees.add(tree);
                }
            }
            // recurse to get the next set of rows from the scanner
            return fetchTrees();
        }
    }
    // start the scanning process
    new AllTreeScanner().fetchTrees();
    return result;
}

Example 18 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class UniqueId method getSuggestScanner.

/**
   * Creates a scanner that scans the right range of rows for suggestions.
   * @param client The HBase client to use.
   * @param tsd_uid_table Table where IDs are stored.
   * @param search The string to start searching at
   * @param kind_or_null The kind of UID to search or null for any kinds.
   * @param max_results The max number of results to return
   */
private static Scanner getSuggestScanner(final HBaseClient client, final byte[] tsd_uid_table, final String search, final byte[] kind_or_null, final int max_results) {
    final byte[] start_row;
    final byte[] end_row;
    if (search.isEmpty()) {
        start_row = START_ROW;
        end_row = END_ROW;
    } else {
        start_row = toBytes(search);
        end_row = Arrays.copyOf(start_row, start_row.length);
        end_row[start_row.length - 1]++;
    }
    final Scanner scanner = client.newScanner(tsd_uid_table);
    scanner.setStartKey(start_row);
    scanner.setStopKey(end_row);
    scanner.setFamily(ID_FAMILY);
    if (kind_or_null != null) {
        scanner.setQualifier(kind_or_null);
    }
    scanner.setMaxNumRows(max_results <= 4096 ? max_results : 4096);
    return scanner;
}

Example 19 with Scanner

use of org.hbase.async.Scanner in project YCSB by brianfrankcooper.

the class AsyncHBaseClient method scan.

@Override
public Status scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String, ByteIterator>> result) {
    setTable(table);
    final Scanner scanner = client.newScanner(lastTableBytes);
    scanner.setFamily(columnFamilyBytes);
    scanner.setStartKey(startkey.getBytes(UTF8_CHARSET));
    // No end key... *sniff*
    if (fields != null) {
        scanner.setQualifiers(getQualifierList(fields));
    }
    // no filters? *sniff*
    ArrayList<ArrayList<KeyValue>> rows = null;
    try {
        int numResults = 0;
        while ((rows = scanner.nextRows().join(joinTimeout)) != null) {
            for (final ArrayList<KeyValue> row : rows) {
                final HashMap<String, ByteIterator> rowResult = new HashMap<String, ByteIterator>(row.size());
                for (final KeyValue column : row) {
                    rowResult.put(new String(column.qualifier()), // be GC'd.
                    new ByteArrayByteIterator(column.value()));
                    if (debug) {
                        System.out.println("Got scan result for key: " + Bytes.pretty(column.key()));
                    }
                }
                result.add(rowResult);
                numResults++;
                if (numResults >= recordcount) {
                    // if hit recordcount, bail out
                    break;
                }
            }
        }
        scanner.close().join(joinTimeout);
        return Status.OK;
    } catch (InterruptedException e) {
        System.err.println("Thread interrupted");
        Thread.currentThread().interrupt();
    } catch (Exception e) {
        System.err.println("Failure reading from row with key " + startkey + ": " + e.getMessage());
        return Status.ERROR;
    }
    return Status.ERROR;
}

Example 20 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class Annotation method getGlobalAnnotations.

/**
   * Scans through the global annotation storage rows and returns a list of 
   * parsed annotation objects. If no annotations were found for the given
   * timespan, the resulting list will be empty.
   * @param tsdb The TSDB to use for storage access
   * @param start_time Start time to scan from. May be 0
   * @param end_time End time to scan to. Must be greater than 0
   * @return A list with detected annotations. May be empty.
   * @throws IllegalArgumentException if the end timestamp has not been set or 
   * the end time is less than the start time
   */
public static Deferred<List<Annotation>> getGlobalAnnotations(final TSDB tsdb, final long start_time, final long end_time) {
    if (end_time < 1) {
        throw new IllegalArgumentException("The end timestamp has not been set");
    }
    if (end_time < start_time) {
        throw new IllegalArgumentException("The end timestamp cannot be less than the start timestamp");
    }
    /**
     * Scanner that loops through the [0, 0, 0, timestamp] rows looking for
     * global annotations. Returns a list of parsed annotation objects.
     * The list may be empty.
     */
    final class ScannerCB implements Callback<Deferred<List<Annotation>>, ArrayList<ArrayList<KeyValue>>> {

        final Scanner scanner;

        final ArrayList<Annotation> annotations = new ArrayList<Annotation>();

        /**
       * Initializes the scanner
       */
        public ScannerCB() {
            final byte[] start = new byte[Const.SALT_WIDTH() + TSDB.metrics_width() + Const.TIMESTAMP_BYTES];
            final byte[] end = new byte[Const.SALT_WIDTH() + TSDB.metrics_width() + Const.TIMESTAMP_BYTES];
            final long normalized_start = (start_time - (start_time % Const.MAX_TIMESPAN));
            final long normalized_end = (end_time - (end_time % Const.MAX_TIMESPAN) + Const.MAX_TIMESPAN);
            Bytes.setInt(start, (int) normalized_start, Const.SALT_WIDTH() + TSDB.metrics_width());
            Bytes.setInt(end, (int) normalized_end, Const.SALT_WIDTH() + TSDB.metrics_width());
            scanner = tsdb.getClient().newScanner(tsdb.dataTable());
            scanner.setStartKey(start);
            scanner.setStopKey(end);
            scanner.setFamily(FAMILY);
        }

        public Deferred<List<Annotation>> scan() {
            return scanner.nextRows().addCallbackDeferring(this);
        }

        @Override
        public Deferred<List<Annotation>> call(final ArrayList<ArrayList<KeyValue>> rows) throws Exception {
            if (rows == null || rows.isEmpty()) {
                return Deferred.fromResult((List<Annotation>) annotations);
            }
            for (final ArrayList<KeyValue> row : rows) {
                for (KeyValue column : row) {
                    if ((column.qualifier().length == 3 || column.qualifier().length == 5) && column.qualifier()[0] == PREFIX()) {
                        Annotation note = JSON.parseToObject(column.value(), Annotation.class);
                        if (note.start_time < start_time || note.end_time > end_time) {
                            continue;
                        }
                        annotations.add(note);
                    }
                }
            }
            return scan();
        }
    }
    return new ScannerCB().scan();
}