Examples with GetRequest org.hbase.async.GetRequest used on opensource projects

Example 16 with GetRequest

use of org.hbase.async.GetRequest in project opentsdb by OpenTSDB.

the class Tree method fetchNotMatched.

/**
 * Returns the not-matched set from storage for the given tree, optionally for
 * only the list of TSUIDs provided.
 * <b>Note:</b> This can potentially be a large list if the rule set was
 * written poorly and there were many timeseries so only call this
 * without a list of TSUIDs if you feel confident the number is small.
 * @param tsdb TSDB to use for storage access
 * @param tree_id ID of the tree to fetch non matches for
 * @param tsuids An optional list of TSUIDs to fetch non-matches for. This may
 * be empty or null, in which case all non-matches for the tree will be
 * returned.
 * @return A list of not-matched mappings or null if nothing was found
 * @throws HBaseException if there was an issue
 * @throws IllegalArgumentException if the tree ID was invalid
 */
public static Deferred<Map<String, String>> fetchNotMatched(final TSDB tsdb, final int tree_id, final List<String> tsuids) {
    if (tree_id < 1 || tree_id > 65535) {
        throw new IllegalArgumentException("Invalid Tree ID");
    }
    final byte[] row_key = new byte[TREE_ID_WIDTH + 1];
    System.arraycopy(idToBytes(tree_id), 0, row_key, 0, TREE_ID_WIDTH);
    row_key[TREE_ID_WIDTH] = NOT_MATCHED_ROW_SUFFIX;
    final GetRequest get = new GetRequest(tsdb.treeTable(), row_key);
    get.family(TREE_FAMILY);
    // of qualifiers so we only fetch those columns.
    if (tsuids != null && !tsuids.isEmpty()) {
        final byte[][] qualifiers = new byte[tsuids.size()][];
        int index = 0;
        for (String tsuid : tsuids) {
            final byte[] qualifier = new byte[NOT_MATCHED_PREFIX.length + (tsuid.length() / 2)];
            System.arraycopy(NOT_MATCHED_PREFIX, 0, qualifier, 0, NOT_MATCHED_PREFIX.length);
            final byte[] tsuid_bytes = UniqueId.stringToUid(tsuid);
            System.arraycopy(tsuid_bytes, 0, qualifier, NOT_MATCHED_PREFIX.length, tsuid_bytes.length);
            qualifiers[index] = qualifier;
            index++;
        }
        get.qualifiers(qualifiers);
    }
    /**
     * Called after issuing the row get request to parse out the results and
     * compile the list of collisions.
     */
    final class GetCB implements Callback<Deferred<Map<String, String>>, ArrayList<KeyValue>> {

        @Override
        public Deferred<Map<String, String>> call(final ArrayList<KeyValue> row) throws Exception {
            if (row == null || row.isEmpty()) {
                final Map<String, String> empty = new HashMap<String, String>(0);
                return Deferred.fromResult(empty);
            }
            Map<String, String> not_matched = new HashMap<String, String>(row.size());
            for (KeyValue column : row) {
                final byte[] parsed_tsuid = Arrays.copyOfRange(column.qualifier(), NOT_MATCHED_PREFIX.length, column.qualifier().length);
                not_matched.put(UniqueId.uidToString(parsed_tsuid), new String(column.value(), CHARSET));
            }
            return Deferred.fromResult(not_matched);
        }
    }
    return tsdb.getClient().get(get).addCallbackDeferring(new GetCB());
}

Example 17 with GetRequest

use of org.hbase.async.GetRequest in project opentsdb by OpenTSDB.

the class UidManager method findAndPrintRow.

/**
 * Gets a given row in HBase and prints it on standard output.
 * @param client The HBase client to use.
 * @param table The name of the HBase table to use.
 * @param key The row key to attempt to get from HBase.
 * @param family The family in which we're interested.
 * @return 0 if at least one cell was found and printed, 1 otherwise.
 */
private static int findAndPrintRow(final HBaseClient client, final byte[] table, final byte[] key, final byte[] family, boolean formard) {
    final GetRequest get = new GetRequest(table, key);
    get.family(family);
    ArrayList<KeyValue> row;
    try {
        row = client.get(get).joinUninterruptibly();
    } catch (HBaseException e) {
        LOG.error("Get failed: " + get, e);
        return 1;
    } catch (Exception e) {
        LOG.error("WTF?  Unexpected exception type, get=" + get, e);
        return 42;
    }
    return printResult(row, family, formard) ? 0 : 1;
}

Example 18 with GetRequest

use of org.hbase.async.GetRequest in project opentsdb by OpenTSDB.

the class Leaf method getFromStorage.

/**
 * Attempts to fetch the requested leaf from storage.
 * <b>Note:</b> This method will not load the UID names from a TSDB. This is
 * only used to fetch a particular leaf from storage for collision detection
 * @param tsdb The TSDB to use for storage access
 * @param branch_id ID of the branch this leaf belongs to
 * @param display_name Name of the leaf
 * @return A valid leaf if found, null if the leaf did not exist
 * @throws HBaseException if there was an issue
 * @throws JSONException if the object could not be serialized
 */
private static Deferred<Leaf> getFromStorage(final TSDB tsdb, final byte[] branch_id, final String display_name) {
    final Leaf leaf = new Leaf();
    leaf.setDisplayName(display_name);
    final GetRequest get = new GetRequest(tsdb.treeTable(), branch_id);
    get.family(Tree.TREE_FAMILY());
    get.qualifier(leaf.columnQualifier());
    /**
     * Called with the results of the fetch from storage
     */
    final class GetCB implements Callback<Deferred<Leaf>, ArrayList<KeyValue>> {

        /**
         * @return null if the row was empty, a valid Leaf if parsing was
         * successful
         */
        @Override
        public Deferred<Leaf> call(ArrayList<KeyValue> row) throws Exception {
            if (row == null || row.isEmpty()) {
                return Deferred.fromResult(null);
            }
            final Leaf leaf = JSON.parseToObject(row.get(0).value(), Leaf.class);
            return Deferred.fromResult(leaf);
        }
    }
    return tsdb.getClient().get(get).addCallbackDeferring(new GetCB());
}

Example 19 with GetRequest

use of org.hbase.async.GetRequest in project opentsdb by OpenTSDB.

the class CliUtils method getMaxMetricID.

/**
 * Returns the max metric ID from the UID table
 * @param tsdb The TSDB to use for data access
 * @return The max metric ID as an integer value, may be 0 if the UID table
 * hasn't been initialized or is missing the UID row or metrics column.
 * @throws IllegalStateException if the UID column can't be found or couldn't
 * be parsed
 */
static long getMaxMetricID(final TSDB tsdb) {
    // first up, we need the max metric ID so we can split up the data table
    // amongst threads.
    final GetRequest get = new GetRequest(tsdb.uidTable(), new byte[] { 0 });
    get.family("id".getBytes(CHARSET));
    get.qualifier("metrics".getBytes(CHARSET));
    ArrayList<KeyValue> row;
    try {
        row = tsdb.getClient().get(get).joinUninterruptibly();
        if (row == null || row.isEmpty()) {
            return 0;
        }
        final byte[] id_bytes = row.get(0).value();
        if (id_bytes.length != 8) {
            throw new IllegalStateException("Invalid metric max UID, wrong # of bytes");
        }
        return Bytes.getLong(id_bytes);
    } catch (Exception e) {
        throw new RuntimeException("Shouldn't be here", e);
    }
}

Example 20 with GetRequest

use of org.hbase.async.GetRequest in project opentsdb by OpenTSDB.

the class MultiGetQuery method prepareConcurrentMultiGetTasks.

/**
 * Compiles the list of TSUIDs and GetRequests to send to execute against
 * storage. Each batch will only have requests for one salt, i.e a batch
 * will not have requests with multiple salts.
 */
@VisibleForTesting
void prepareConcurrentMultiGetTasks() {
    multi_get_wait_cnt = 0;
    prepare_multi_get_start_time = DateTime.currentTimeMillis();
    final List<Long> row_base_time_list;
    if (RollupQuery.isValidQuery(rollup_query)) {
        row_base_time_list = prepareRowBaseTimesRollup();
    } else {
        row_base_time_list = prepareRowBaseTimes();
    }
    int next_concurrency_index = 0;
    List<GetRequest> gets_to_prepare = new ArrayList<GetRequest>(batch_size);
    Set<byte[]> tsuids = new ByteSet();
    final ByteMap<ByteMap<List<GetRequest>>> all_tsuids_gets;
    if (multiget_no_meta) {
        // prepare the tagvs combinations and base time list
        final List<byte[][]> tagv_compinations = prepareAllTagvCompounds();
        all_tsuids_gets = prepareRequestsNoMeta(tagv_compinations, row_base_time_list);
    } else {
        all_tsuids_gets = prepareRequests(row_base_time_list, tags);
    }
    // Iterate over all salts
    for (final Entry<byte[], ByteMap<List<GetRequest>>> salts_entry : all_tsuids_gets.entrySet()) {
        if (gets_to_prepare.size() > 0) {
            // if we have any gets_to_prepare for previous salt, create a
            // request out of it.
            final MultiGetTask task = new MultiGetTask(tsuids, gets_to_prepare);
            final List<MultiGetTask> mulget_task_list = multi_get_tasks.get((next_concurrency_index++) % concurrency_multi_get);
            mulget_task_list.add(task);
            ++multi_get_wait_cnt;
            multi_get_num_get_requests = multi_get_num_get_requests + gets_to_prepare.size();
            gets_to_prepare = new ArrayList<GetRequest>(batch_size);
            tsuids = new ByteSet();
        }
        byte[] curr_salt = salts_entry.getKey();
        // Iterate over all tsuid's in curr_salt and add them to gets_to_prepare
        for (final Entry<byte[], List<GetRequest>> gets_entry : salts_entry.getValue()) {
            byte[] tsuid = gets_entry.getKey();
            for (GetRequest request : gets_entry.getValue()) {
                if (gets_to_prepare.size() >= batch_size) {
                    // close batch and create a MultiGetTask
                    final MultiGetTask task = new MultiGetTask(tsuids, gets_to_prepare);
                    final List<MultiGetTask> mulget_task_list = multi_get_tasks.get((next_concurrency_index++) % concurrency_multi_get);
                    mulget_task_list.add(task);
                    ++multi_get_wait_cnt;
                    multi_get_num_get_requests = multi_get_num_get_requests + gets_to_prepare.size();
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Finished preparing MultiGetRequest with " + gets_to_prepare.size() + " requests for salt " + curr_salt + " for tsuid " + Bytes.pretty(tsuid));
                    }
                    // prepare a new task list and tsuids
                    gets_to_prepare = new ArrayList<GetRequest>(batch_size);
                    tsuids = new ByteSet();
                }
                gets_to_prepare.add(request);
                tsuids.add(gets_entry.getKey());
            }
            tsuids.add(gets_entry.getKey());
        }
    }
    if (gets_to_prepare.size() > 0) {
        final MultiGetTask task = new MultiGetTask(tsuids, gets_to_prepare);
        final List<MultiGetTask> mulget_task_list = multi_get_tasks.get((next_concurrency_index++) % concurrency_multi_get);
        mulget_task_list.add(task);
        ++multi_get_wait_cnt;
        multi_get_num_get_requests = multi_get_num_get_requests + gets_to_prepare.size();
        LOG.debug("Finished preparing MultiGetRequest with " + gets_to_prepare.size());
        gets_to_prepare = new ArrayList<GetRequest>(batch_size);
        tsuids = new ByteSet();
    }
    prepare_multi_get_end_time = DateTime.currentTimeMillis();
    if (LOG.isDebugEnabled()) {
        LOG.debug("Finished preparing concurrency multi get task with " + multi_get_wait_cnt + " tasks using " + (prepare_multi_get_end_time - prepare_multi_get_start_time) + "ms");
    }
}