Examples with Scanner org.hbase.async.Scanner used on opensource projects

Example 31 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class DumpSeries method doDump.

private static void doDump(final TSDB tsdb, final HBaseClient client, final byte[] table, final boolean delete, final boolean importformat, final String[] args) throws Exception {
    final ArrayList<Query> queries = new ArrayList<Query>();
    CliQuery.parseCommandLineQuery(args, tsdb, queries, null, null);
    final StringBuilder buf = new StringBuilder();
    for (final Query query : queries) {
        final List<Scanner> scanners = Internal.getScanners(query);
        for (Scanner scanner : scanners) {
            ArrayList<ArrayList<KeyValue>> rows;
            while ((rows = scanner.nextRows().joinUninterruptibly()) != null) {
                for (final ArrayList<KeyValue> row : rows) {
                    buf.setLength(0);
                    final byte[] key = row.get(0).key();
                    final long base_time = Internal.baseTime(tsdb, key);
                    final String metric = Internal.metricName(tsdb, key);
                    // Print the row key.
                    if (!importformat) {
                        buf.append(Arrays.toString(key)).append(' ').append(metric).append(' ').append(base_time).append(" (").append(date(base_time)).append(") ");
                        try {
                            buf.append(Internal.getTags(tsdb, key));
                        } catch (RuntimeException e) {
                            buf.append(e.getClass().getName() + ": " + e.getMessage());
                        }
                        buf.append('\n');
                        System.out.print(buf);
                    }
                    // Print individual cells.
                    buf.setLength(0);
                    if (!importformat) {
                        buf.append("  ");
                    }
                    for (final KeyValue kv : row) {
                        // Discard everything or keep initial spaces.
                        buf.setLength(importformat ? 0 : 2);
                        formatKeyValue(buf, tsdb, importformat, kv, base_time, metric);
                        if (buf.length() > 0) {
                            buf.append('\n');
                            System.out.print(buf);
                        }
                    }
                    if (delete) {
                        final DeleteRequest del = new DeleteRequest(table, key);
                        client.delete(del);
                    }
                }
            }
        }
    }
}

Example 32 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class Tree method setupAllTreeScanner.

/**
   * Configures a scanner to run through all rows in the UID table that are
   * {@link #TREE_ID_WIDTH} bytes wide using a row key regex filter 
   * @param tsdb The TSDB to use for storage access
   * @return The configured HBase scanner
   */
private static Scanner setupAllTreeScanner(final TSDB tsdb) {
    final byte[] start = new byte[TREE_ID_WIDTH];
    final byte[] end = new byte[TREE_ID_WIDTH];
    Arrays.fill(end, (byte) 0xFF);
    final Scanner scanner = tsdb.getClient().newScanner(tsdb.treeTable());
    scanner.setStartKey(start);
    scanner.setStopKey(end);
    scanner.setFamily(TREE_FAMILY);
    // set the filter to match only on TREE_ID_WIDTH row keys
    final StringBuilder buf = new StringBuilder(20);
    buf.append(// Ensure we use the DOTALL flag.
    "(?s)" + "^\\Q");
    buf.append("\\E(?:.{").append(TREE_ID_WIDTH).append("})$");
    scanner.setKeyRegexp(buf.toString(), CHARSET);
    return scanner;
}

Example 33 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class Tree method deleteTree.

/**
   * Attempts to delete all branches, leaves, collisions and not-matched entries
   * for the given tree. Optionally can delete the tree definition and rules as
   * well.
   * <b>Warning:</b> This call can take a long time to complete so it should
   * only be done from a command line or issues once via RPC and allowed to
   * process. Multiple deletes running at the same time on the same tree
   * shouldn't be an issue but it's a waste of resources.
   * @param tsdb The TSDB to use for storage access
   * @param tree_id ID of the tree to delete
   * @param delete_definition Whether or not the tree definition and rule set
   * should be deleted as well
   * @return True if the deletion completed successfully, false if there was an
   * issue.
   * @throws HBaseException if there was an issue
   * @throws IllegalArgumentException if the tree ID was invalid
   */
public static Deferred<Boolean> deleteTree(final TSDB tsdb, final int tree_id, final boolean delete_definition) {
    if (tree_id < 1 || tree_id > 65535) {
        throw new IllegalArgumentException("Invalid Tree ID");
    }
    // scan all of the rows starting with the tree ID. We can't just delete the
    // rows as there may be other types of data. Thus we have to check the
    // qualifiers of every column to see if it's safe to delete
    final byte[] start = idToBytes(tree_id);
    final byte[] end = idToBytes(tree_id + 1);
    final Scanner scanner = tsdb.getClient().newScanner(tsdb.treeTable());
    scanner.setStartKey(start);
    scanner.setStopKey(end);
    scanner.setFamily(TREE_FAMILY);
    final Deferred<Boolean> completed = new Deferred<Boolean>();
    /**
     * Scanner callback that loops through all rows between tree id and 
     * tree id++ searching for tree related columns to delete.
     */
    final class DeleteTreeScanner implements Callback<Deferred<Boolean>, ArrayList<ArrayList<KeyValue>>> {

        // list where we'll store delete requests for waiting on
        private final ArrayList<Deferred<Object>> delete_deferreds = new ArrayList<Deferred<Object>>();

        /**
       * Fetches the next set of rows from the scanner and adds this class as
       * a callback
       * @return The list of delete requests when the scanner returns a null set
       */
        public Deferred<Boolean> deleteTree() {
            return scanner.nextRows().addCallbackDeferring(this);
        }

        @Override
        public Deferred<Boolean> call(ArrayList<ArrayList<KeyValue>> rows) throws Exception {
            if (rows == null) {
                completed.callback(true);
                return null;
            }
            for (final ArrayList<KeyValue> row : rows) {
                // one delete request per row. We'll almost always delete the whole
                // row, so just preallocate the entire row.
                ArrayList<byte[]> qualifiers = new ArrayList<byte[]>(row.size());
                for (KeyValue column : row) {
                    // tree
                    if (delete_definition && Bytes.equals(TREE_QUALIFIER, column.qualifier())) {
                        LOG.trace("Deleting tree defnition in row: " + Branch.idToString(column.key()));
                        qualifiers.add(column.qualifier());
                    // branches
                    } else if (Bytes.equals(Branch.BRANCH_QUALIFIER(), column.qualifier())) {
                        LOG.trace("Deleting branch in row: " + Branch.idToString(column.key()));
                        qualifiers.add(column.qualifier());
                    // leaves
                    } else if (column.qualifier().length > Leaf.LEAF_PREFIX().length && Bytes.memcmp(Leaf.LEAF_PREFIX(), column.qualifier(), 0, Leaf.LEAF_PREFIX().length) == 0) {
                        LOG.trace("Deleting leaf in row: " + Branch.idToString(column.key()));
                        qualifiers.add(column.qualifier());
                    // collisions
                    } else if (column.qualifier().length > COLLISION_PREFIX.length && Bytes.memcmp(COLLISION_PREFIX, column.qualifier(), 0, COLLISION_PREFIX.length) == 0) {
                        LOG.trace("Deleting collision in row: " + Branch.idToString(column.key()));
                        qualifiers.add(column.qualifier());
                    // not matched
                    } else if (column.qualifier().length > NOT_MATCHED_PREFIX.length && Bytes.memcmp(NOT_MATCHED_PREFIX, column.qualifier(), 0, NOT_MATCHED_PREFIX.length) == 0) {
                        LOG.trace("Deleting not matched in row: " + Branch.idToString(column.key()));
                        qualifiers.add(column.qualifier());
                    // tree rule
                    } else if (delete_definition && column.qualifier().length > TreeRule.RULE_PREFIX().length && Bytes.memcmp(TreeRule.RULE_PREFIX(), column.qualifier(), 0, TreeRule.RULE_PREFIX().length) == 0) {
                        LOG.trace("Deleting tree rule in row: " + Branch.idToString(column.key()));
                        qualifiers.add(column.qualifier());
                    }
                }
                if (qualifiers.size() > 0) {
                    final DeleteRequest delete = new DeleteRequest(tsdb.treeTable(), row.get(0).key(), TREE_FAMILY, qualifiers.toArray(new byte[qualifiers.size()][]));
                    delete_deferreds.add(tsdb.getClient().delete(delete));
                }
            }
            /**
         * Callback used as a kind of buffer so that we don't wind up loading
         * thousands or millions of delete requests into memory and possibly run
         * into a StackOverflowError or general OOM. The scanner defaults are
         * our limit so each pass of the scanner will wait for the previous set
         * of deferreds to complete before continuing
         */
            final class ContinueCB implements Callback<Deferred<Boolean>, ArrayList<Object>> {

                public Deferred<Boolean> call(ArrayList<Object> objects) {
                    LOG.debug("Purged [" + objects.size() + "] columns, continuing");
                    delete_deferreds.clear();
                    // call ourself again to get the next set of rows from the scanner
                    return deleteTree();
                }
            }
            // call ourself again after waiting for the existing delete requests 
            // to complete
            Deferred.group(delete_deferreds).addCallbackDeferring(new ContinueCB());
            return null;
        }
    }
    // start the scanner
    new DeleteTreeScanner().deleteTree();
    return completed;
}

Example 34 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class UniqueId method preloadUidCache.

/**
   * Pre-load UID caches, scanning up to "tsd.core.preload_uid_cache.max_entries"
   * rows from the UID table.
   * @param tsdb The TSDB to use 
   * @param uid_cache_map A map of {@link UniqueId} objects keyed on the kind.
   * @throws HBaseException Passes any HBaseException from HBase scanner.
   * @throws RuntimeException Wraps any non HBaseException from HBase scanner.
   * @2.1
   */
public static void preloadUidCache(final TSDB tsdb, final ByteMap<UniqueId> uid_cache_map) throws HBaseException {
    int max_results = tsdb.getConfig().getInt("tsd.core.preload_uid_cache.max_entries");
    LOG.info("Preloading uid cache with max_results=" + max_results);
    if (max_results <= 0) {
        return;
    }
    Scanner scanner = null;
    try {
        int num_rows = 0;
        scanner = getSuggestScanner(tsdb.getClient(), tsdb.uidTable(), "", null, max_results);
        for (ArrayList<ArrayList<KeyValue>> rows = scanner.nextRows().join(); rows != null; rows = scanner.nextRows().join()) {
            for (final ArrayList<KeyValue> row : rows) {
                for (KeyValue kv : row) {
                    final String name = fromBytes(kv.key());
                    final byte[] kind = kv.qualifier();
                    final byte[] id = kv.value();
                    LOG.debug("id='{}', name='{}', kind='{}'", Arrays.toString(id), name, fromBytes(kind));
                    UniqueId uid_cache = uid_cache_map.get(kind);
                    if (uid_cache != null) {
                        uid_cache.cacheMapping(name, id);
                    }
                }
                num_rows += row.size();
                // free()
                row.clear();
                if (num_rows >= max_results) {
                    break;
                }
            }
        }
        for (UniqueId unique_id_table : uid_cache_map.values()) {
            LOG.info("After preloading, uid cache '{}' has {} ids and {} names.", unique_id_table.kind(), unique_id_table.id_cache.size(), unique_id_table.name_cache.size());
        }
    } catch (Exception e) {
        if (e instanceof HBaseException) {
            throw (HBaseException) e;
        } else if (e instanceof RuntimeException) {
            throw (RuntimeException) e;
        } else {
            throw new RuntimeException("Error while preloading IDs", e);
        }
    } finally {
        if (scanner != null) {
            scanner.close();
        }
    }
}

Example 35 with Scanner

use of org.hbase.async.Scanner in project opentsdb by OpenTSDB.

the class TestUniqueId method suggestWithNoMatch.

@Test
public void suggestWithNoMatch() {
    uid = new UniqueId(client, table, METRIC, 3);
    final Scanner fake_scanner = mock(Scanner.class);
    when(client.newScanner(table)).thenReturn(fake_scanner);
    when(fake_scanner.nextRows()).thenReturn(Deferred.<ArrayList<ArrayList<KeyValue>>>fromResult(null));
    // Watch this! ______,^   I'm writing C++ in Java!
    final List<String> suggestions = uid.suggest("nomatch");
    // No results.
    assertEquals(0, suggestions.size());
    verify(fake_scanner).setStartKey("nomatch".getBytes());
    verify(fake_scanner).setStopKey("nomatci".getBytes());
    verify(fake_scanner).setFamily(ID);
    verify(fake_scanner).setQualifier(METRIC_ARRAY);
}