Examples with GetRequest org.hbase.async.GetRequest used on opensource projects

Example 21 with GetRequest

use of org.hbase.async.GetRequest in project opentsdb by OpenTSDB.

the class MultiGetQuery method prepareRequestsNoMeta.

/**
 * Generates a map of TSUIDs to get requests given the tag permutations.
 * If all salts is enabled, each TSUID will have Const.SALT_BUCKETS() number
 * of entries. Otherwise each TSUID will have one row key.
 * @param tagv_compounds The permutations of tag key and value combinations to
 * search for.
 * @param base_time_list The list of base timestamps.
 * @return A non-null map of TSUIDs to lists of get requests to send to HBase.
 */
@VisibleForTesting
ByteMap<ByteMap<List<GetRequest>>> prepareRequestsNoMeta(final List<byte[][]> tagv_compounds, final List<Long> base_time_list) {
    final int row_size = (Const.SALT_WIDTH() + tsdb.metrics.width() + Const.TIMESTAMP_BYTES + (tsdb.tag_names.width() * tags.get(0).size()) + (tsdb.tag_values.width() * tags.get(0).size()));
    final ByteMap<List<GetRequest>> tsuid_rows = new ByteMap<List<GetRequest>>();
    for (final byte[][] tagvs : tagv_compounds) {
        // TSUID's don't have salts
        // TODO: we reallly don't have to allocate tsuid's here.
        // we can use the row_key array to fetch the tsuid.
        // This will just double the memory utilization per time series.
        final byte[] tsuid = new byte[tsdb.metrics.width() + (tags.get(0).size() * tsdb.tag_names.width()) + tags.get(0).size() * tsdb.tag_values.width()];
        final byte[] row_key = new byte[row_size];
        // metric
        System.arraycopy(metric, 0, row_key, Const.SALT_WIDTH(), tsdb.metrics.width());
        System.arraycopy(metric, 0, tsuid, 0, tsdb.metrics.width());
        final List<GetRequest> rows = new ArrayList<GetRequest>(base_time_list.size());
        // copy tagks and tagvs to the row key
        int tag_index = 0;
        int row_key_copy_offset = Const.SALT_WIDTH() + tsdb.metrics.width() + Const.TIMESTAMP_BYTES;
        int tsuid_copy_offset = tsdb.metrics.width();
        for (Map.Entry<byte[], byte[][]> tag : tags.get(0)) {
            // tagk
            byte[] tagk = tag.getKey();
            System.arraycopy(tagk, 0, row_key, row_key_copy_offset, tsdb.tag_names.width());
            System.arraycopy(tagk, 0, tsuid, tsuid_copy_offset, tsdb.tag_names.width());
            row_key_copy_offset += tsdb.tag_names.width();
            tsuid_copy_offset += tsdb.tag_names.width();
            // tagv
            System.arraycopy(tagvs[tag_index], 0, row_key, row_key_copy_offset, tsdb.tag_values.width());
            System.arraycopy(tagvs[tag_index], 0, tsuid, tsuid_copy_offset, tsdb.tag_values.width());
            row_key_copy_offset += tsdb.tag_values.width();
            tsuid_copy_offset += tsdb.tag_values.width();
            // move to the next tag
            ++tag_index;
        }
        // timestamp.
        for (final long row_base_time : base_time_list) {
            final byte[] key_copy = Arrays.copyOf(row_key, row_key.length);
            // base time
            Internal.setBaseTime(key_copy, (int) row_base_time);
            if (get_all_salts) {
                for (int i = 0; i < Const.SALT_BUCKETS(); i++) {
                    final byte[] copy = Arrays.copyOf(key_copy, key_copy.length);
                    // TODO - handle multi byte salts
                    copy[0] = (byte) i;
                    rows.add(new GetRequest(table_to_fetch, copy, TSDB.FAMILY));
                }
            } else {
                // salt
                RowKey.prefixKeyWithSalt(key_copy);
                rows.add(new GetRequest(table_to_fetch, key_copy, TSDB.FAMILY));
            }
        }
        // end for
        tsuid_rows.put(tsuid, rows);
    }
    // end for
    ByteMap<ByteMap<List<GetRequest>>> return_obj = new ByteMap<ByteMap<List<GetRequest>>>();
    // byte[] salt = new byte[Const.SALT_WIDTH()];
    // System.arraycopy("1".getBytes(), 0, salt, 0, Const.SALT_WIDTH());
    return_obj.put("0".getBytes(), tsuid_rows);
    return return_obj;
}

Example 22 with GetRequest

use of org.hbase.async.GetRequest in project opentsdb by OpenTSDB.

the class TestMultiGetQuery method prepareConcurrentMultiSortedSalts.

@Test
public void prepareConcurrentMultiSortedSalts() throws Exception {
    config.overrideConfig("tsd.query.multi_get.concurrent", "2");
    config.overrideConfig("tsd.query.multi_get.batch_size", "2");
    Const.setSaltWidth(1);
    MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, 0, max_bytes, false, multiget_no_meta);
    mgq.prepareConcurrentMultiGetTasks();
    final List<List<MultiGetTask>> tasks = mgq.getMultiGetTasks();
    assertEquals(config.getInt("tsd.query.multi_get.concurrent"), tasks.size());
    for (List<MultiGetTask> taskList : tasks) {
        for (MultiGetTask task : taskList) {
            byte salt = task.getGets().get(0).key()[0];
            for (GetRequest request : task.getGets()) {
                assertEquals(salt, request.key()[0]);
            }
        }
    }
    Const.setSaltWidth(0);
}

Example 23 with GetRequest

use of org.hbase.async.GetRequest in project opentsdb by OpenTSDB.

the class UniqueId method hbaseGet.

/**
 * Returns the cell of the specified row key, using family:kind.
 */
private Deferred<byte[]> hbaseGet(final byte[] key, final byte[] family) {
    final GetRequest get = new GetRequest(table, key);
    get.family(family).qualifier(kind);
    class GetCB implements Callback<byte[], ArrayList<KeyValue>> {

        public byte[] call(final ArrayList<KeyValue> row) {
            if (row == null || row.isEmpty()) {
                return null;
            }
            return row.get(0).value();
        }
    }
    return client.get(get).addCallback(new GetCB());
}

Example 24 with GetRequest

use of org.hbase.async.GetRequest in project opentsdb by OpenTSDB.

the class UniqueId method getUsedUIDs.

/**
 * Returns a map of max UIDs from storage for the given list of UID types
 * @param tsdb The TSDB to which we belong
 * @param kinds A list of qualifiers to fetch
 * @return A map with the "kind" as the key and the maximum assigned UID as
 * the value
 * @since 2.0
 */
public static Deferred<Map<String, Long>> getUsedUIDs(final TSDB tsdb, final byte[][] kinds) {
    /**
     * Returns a map with 0 if the max ID row hasn't been initialized yet,
     * otherwise the map has actual data
     */
    final class GetCB implements Callback<Map<String, Long>, ArrayList<KeyValue>> {

        @Override
        public Map<String, Long> call(final ArrayList<KeyValue> row) throws Exception {
            final Map<String, Long> results = new HashMap<String, Long>(3);
            if (row == null || row.isEmpty()) {
                // it could be the case that this is the first time the TSD has run
                // and the user hasn't put any metrics in, so log and return 0s
                LOG.info("Could not find the UID assignment row");
                for (final byte[] kind : kinds) {
                    results.put(new String(kind, CHARSET), 0L);
                }
                return results;
            }
            for (final KeyValue column : row) {
                results.put(new String(column.qualifier(), CHARSET), Bytes.getLong(column.value()));
            }
            // for missing columns
            for (final byte[] kind : kinds) {
                if (results.get(new String(kind, CHARSET)) == null) {
                    results.put(new String(kind, CHARSET), 0L);
                }
            }
            return results;
        }
    }
    final GetRequest get = new GetRequest(tsdb.uidTable(), MAXID_ROW);
    get.family(ID_FAMILY);
    get.qualifiers(kinds);
    return tsdb.getClient().get(get).addCallback(new GetCB());
}

Example 25 with GetRequest

use of org.hbase.async.GetRequest in project opentsdb by OpenTSDB.

the class TSMeta method getFromStorage.

/**
 * Attempts to fetch the timeseries meta data from storage.
 * This method will fetch the {@code counter} and {@code meta} columns.
 * <b>Note:</b> This method will not load the UIDMeta objects.
 * @param tsdb The TSDB to use for storage access
 * @param tsuid The UID of the meta to fetch
 * @return A TSMeta object if found, null if not
 * @throws HBaseException if there was an issue fetching
 * @throws IllegalArgumentException if parsing failed
 * @throws JSONException if the data was corrupted
 */
private static Deferred<TSMeta> getFromStorage(final TSDB tsdb, final byte[] tsuid) {
    /**
     * Called after executing the GetRequest to parse the meta data.
     */
    final class GetCB implements Callback<Deferred<TSMeta>, ArrayList<KeyValue>> {

        /**
         * @return Null if the meta did not exist or a valid TSMeta object if it
         * did.
         */
        @Override
        public Deferred<TSMeta> call(final ArrayList<KeyValue> row) throws Exception {
            if (row == null || row.isEmpty()) {
                return Deferred.fromResult(null);
            }
            long dps = 0;
            long last_received = 0;
            TSMeta meta = null;
            for (KeyValue column : row) {
                if (Arrays.equals(COUNTER_QUALIFIER, column.qualifier())) {
                    dps = Bytes.getLong(column.value());
                    last_received = column.timestamp() / 1000;
                } else if (Arrays.equals(META_QUALIFIER, column.qualifier())) {
                    meta = JSON.parseToObject(column.value(), TSMeta.class);
                }
            }
            if (meta == null) {
                LOG.warn("Found a counter TSMeta column without a meta for TSUID: " + UniqueId.uidToString(row.get(0).key()));
                return Deferred.fromResult(null);
            }
            meta.total_dps = dps;
            meta.last_received = last_received;
            return Deferred.fromResult(meta);
        }
    }
    final GetRequest get = new GetRequest(tsdb.metaTable(), tsuid);
    get.family(FAMILY);
    get.qualifiers(new byte[][] { COUNTER_QUALIFIER, META_QUALIFIER });
    return tsdb.getClient().get(get).addCallbackDeferring(new GetCB());
}