Examples with GetRequest org.hbase.async.GetRequest used on opensource projects

Example 6 with GetRequest

use of org.hbase.async.GetRequest in project opentsdb by OpenTSDB.

the class TestMultiGetQuery method prepareRequests.

@Test
public void prepareRequests() throws Exception {
    MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, 0, max_bytes, false, multiget_no_meta);
    List<Long> timestamps = mgq.prepareRowBaseTimes();
    ByteMap<ByteMap<List<GetRequest>>> row_map = mgq.prepareRequests(timestamps, q_tags);
    ByteSet tsuids = new ByteSet();
    for (ByteMap<List<GetRequest>> rows : row_map.values()) {
        tsuids.addAll(rows.keySet());
    }
    assertEquals(3, tsuids.size());
    List<GetRequest> rows = new ArrayList<GetRequest>();
    for (Entry<byte[], ByteMap<List<GetRequest>>> salt_entry : row_map.entrySet()) {
        System.out.println(salt_entry.getValue());
        rows.addAll(salt_entry.getValue().get(getTSUID(METRIC_STRING, TAGK_STRING, "A", TAGK_B_STRING, "D")));
    }
    assertEquals(timestamps.size(), rows.size());
    for (int i = 0; i < timestamps.size(); i++) {
        byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), TAGK_STRING, "A", TAGK_B_STRING, "D");
        assertArrayEquals(key, rows.get(i).key());
    }
    rows = new ArrayList<GetRequest>();
    for (Entry<byte[], ByteMap<List<GetRequest>>> salt_entry : row_map.entrySet()) {
        rows.addAll(salt_entry.getValue().get(getTSUID(METRIC_STRING, TAGK_STRING, "B", TAGK_B_STRING, "D")));
    }
    assertEquals(timestamps.size(), rows.size());
    for (int i = 0; i < timestamps.size(); i++) {
        byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), TAGK_STRING, "B", TAGK_B_STRING, "D");
        assertArrayEquals(key, rows.get(i).key());
    }
    rows = new ArrayList<GetRequest>();
    for (Entry<byte[], ByteMap<List<GetRequest>>> salt_entry : row_map.entrySet()) {
        rows.addAll(salt_entry.getValue().get(getTSUID(METRIC_STRING, TAGK_STRING, "C", TAGK_B_STRING, "D")));
    }
    assertEquals(timestamps.size(), rows.size());
    for (int i = 0; i < timestamps.size(); i++) {
        byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), TAGK_STRING, "C", TAGK_B_STRING, "D");
        assertArrayEquals(key, rows.get(i).key());
    }
    rows = new ArrayList<GetRequest>();
    for (Entry<byte[], ByteMap<List<GetRequest>>> salt_entry : row_map.entrySet()) {
        rows.addAll(salt_entry.getValue().get(getTSUID(METRIC_STRING, TAGK_STRING, "A", TAGK_B_STRING, "D")));
    }
    assertEquals(timestamps.size(), rows.size());
    for (int i = 0; i < timestamps.size(); i++) {
        byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), TAGK_STRING, "A", TAGK_B_STRING, "D");
        assertArrayEquals(key, rows.get(i).key());
    }
    rows = new ArrayList<GetRequest>();
    for (Entry<byte[], ByteMap<List<GetRequest>>> salt_entry : row_map.entrySet()) {
        rows.addAll(salt_entry.getValue().get(getTSUID(METRIC_STRING, TAGK_STRING, "B", TAGK_B_STRING, "D")));
    }
    assertEquals(timestamps.size(), rows.size());
    for (int i = 0; i < timestamps.size(); i++) {
        byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), TAGK_STRING, "B", TAGK_B_STRING, "D");
        assertArrayEquals(key, rows.get(i).key());
    }
    rows = new ArrayList<GetRequest>();
    for (Entry<byte[], ByteMap<List<GetRequest>>> salt_entry : row_map.entrySet()) {
        rows.addAll(salt_entry.getValue().get(getTSUID(METRIC_STRING, TAGK_STRING, "C", TAGK_B_STRING, "D")));
    }
    assertEquals(timestamps.size(), rows.size());
    for (int i = 0; i < timestamps.size(); i++) {
        byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), TAGK_STRING, "C", TAGK_B_STRING, "D");
        assertArrayEquals(key, rows.get(i).key());
    }
}

Example 7 with GetRequest

use of org.hbase.async.GetRequest in project opentsdb by OpenTSDB.

the class TestMultiGetQuery method prepareConcurrentMultiGetTasksSmallBatchAndSmallConcurrent.

@Test
public void prepareConcurrentMultiGetTasksSmallBatchAndSmallConcurrent() throws Exception {
    config.overrideConfig("tsd.query.multi_get.concurrent", "2");
    config.overrideConfig("tsd.query.multi_get.batch_size", "17");
    MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, 0, max_bytes, false, multiget_no_meta);
    mgq.prepareConcurrentMultiGetTasks();
    final List<List<MultiGetTask>> tasks = mgq.getMultiGetTasks();
    assertEquals(config.getInt("tsd.query.multi_get.concurrent"), tasks.size());
    assertEquals(2, tasks.get(0).size());
    assertEquals(1, tasks.get(1).size());
    for (List<MultiGetTask> taskList : tasks) {
        for (MultiGetTask task : taskList) {
            byte salt = task.getGets().get(0).key()[0];
            for (GetRequest request : task.getGets()) {
                assertEquals(salt, request.key()[0]);
            }
        }
    }
    // first batch
    MultiGetTask task = tasks.get(0).get(0);
    assertEquals(1, task.getTSUIDs().size());
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "A", TAGK_B_STRING, "D")));
    assertEquals(17, task.getGets().size());
    int idx = 0;
    int ts = 1481227200;
    while (idx < 16) {
        // notice the early cut off. The last hour should spill over.
        assertArrayEquals(task.getGets().get(idx++).key(), getRowKey(METRIC_STRING, ts, TAGK_STRING, "A", TAGK_B_STRING, "D"));
        ts += 3600;
    }
    // 17th request
    assertArrayEquals(task.getGets().get(idx++).key(), getRowKey(METRIC_STRING, ts, TAGK_STRING, "A", TAGK_B_STRING, "D"));
    ts += 3600;
    // next batch
    task = tasks.get(1).get(0);
    assertEquals(1, task.getTSUIDs().size());
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "B", TAGK_B_STRING, "D")));
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "B", TAGK_B_STRING, "D")));
    assertEquals(17, task.getGets().size());
    idx = 0;
    ts = 1481227200;
    while (idx < 17) {
        assertArrayEquals(task.getGets().get(idx++).key(), getRowKey(METRIC_STRING, ts, TAGK_STRING, "B", TAGK_B_STRING, "D"));
        ts += 3600;
    }
    // next batch
    task = tasks.get(0).get(1);
    assertEquals(1, task.getTSUIDs().size());
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "C", TAGK_B_STRING, "D")));
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "C", TAGK_B_STRING, "D")));
    assertEquals(17, task.getGets().size());
    idx = 0;
    ts = 1481227200;
    while (idx < 17) {
        assertArrayEquals(task.getGets().get(idx++).key(), getRowKey(METRIC_STRING, ts, TAGK_STRING, "C", TAGK_B_STRING, "D"));
        ts += 3600;
    }
}

Example 8 with GetRequest

use of org.hbase.async.GetRequest in project opentsdb by OpenTSDB.

the class TestMultiGetQuery method prepareConcurrentMultiGetTasks.

@Test
public void prepareConcurrentMultiGetTasks() throws Exception {
    MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, 0, max_bytes, false, multiget_no_meta);
    mgq.prepareConcurrentMultiGetTasks();
    final List<List<MultiGetTask>> tasks = mgq.getMultiGetTasks();
    assertEquals(config.getInt("tsd.query.multi_get.concurrent"), tasks.size());
    for (int i = 1; i < tasks.size(); i++) {
        assertTrue(tasks.get(i).isEmpty());
    }
    for (List<MultiGetTask> taskList : tasks) {
        for (MultiGetTask task : taskList) {
            byte salt = task.getGets().get(0).key()[0];
            for (GetRequest request : task.getGets()) {
                assertEquals(salt, request.key()[0]);
            }
        }
    }
    assertEquals(1, tasks.get(0).size());
    MultiGetTask task = tasks.get(0).get(0);
    assertEquals(3, task.getTSUIDs().size());
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "A", TAGK_B_STRING, "D")));
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "B", TAGK_B_STRING, "D")));
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "C", TAGK_B_STRING, "D")));
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "A", TAGK_B_STRING, "D")));
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "B", TAGK_B_STRING, "D")));
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "C", TAGK_B_STRING, "D")));
    assertEquals(51, task.getGets().size());
    // 6 sets of 17 timestamps. Ugly UT
    int idx = 0;
    int ts = 1481227200;
    while (idx < 17) {
        assertArrayEquals(task.getGets().get(idx++).key(), getRowKey(METRIC_STRING, ts, TAGK_STRING, "A", TAGK_B_STRING, "D"));
        ts += 3600;
    }
    ts = 1481227200;
    while (idx < 17) {
        assertArrayEquals(task.getGets().get(idx++).key(), getRowKey(METRIC_STRING, ts, TAGK_STRING, "B", TAGK_B_STRING, "D"));
        ts += 3600;
    }
    ts = 1481227200;
    while (idx < 17) {
        assertArrayEquals(task.getGets().get(idx++).key(), getRowKey(METRIC_STRING, ts, TAGK_STRING, "C", TAGK_B_STRING, "D"));
        ts += 3600;
    }
}

Example 9 with GetRequest

use of org.hbase.async.GetRequest in project opentsdb by OpenTSDB.

the class Annotation method getAnnotation.

/**
 * Attempts to fetch a global or local annotation from storage
 * @param tsdb The TSDB to use for storage access
 * @param tsuid The TSUID as a byte array. May be null if retrieving a global
 * annotation
 * @param start_time The start time as a Unix epoch timestamp
 * @return A valid annotation object if found, null if not
 */
public static Deferred<Annotation> getAnnotation(final TSDB tsdb, final byte[] tsuid, final long start_time) {
    /**
     * Called after executing the GetRequest to parse the meta data.
     */
    final class GetCB implements Callback<Deferred<Annotation>, ArrayList<KeyValue>> {

        /**
         * @return Null if the meta did not exist or a valid Annotation object if
         * it did.
         */
        @Override
        public Deferred<Annotation> call(final ArrayList<KeyValue> row) throws Exception {
            if (row == null || row.isEmpty()) {
                return Deferred.fromResult(null);
            }
            Annotation note = JSON.parseToObject(row.get(0).value(), Annotation.class);
            return Deferred.fromResult(note);
        }
    }
    final GetRequest get = new GetRequest(tsdb.dataTable(), getRowKey(start_time, tsuid));
    get.family(FAMILY);
    get.qualifier(getQualifier(start_time));
    return tsdb.getClient().get(get).addCallbackDeferring(new GetCB());
}

Example 10 with GetRequest

use of org.hbase.async.GetRequest in project opentsdb by OpenTSDB.

the class TSMeta method counterExistsInStorage.

/**
 * Determines if the counter column exists for the TSUID.
 * This is used by the UID Manager tool to determine if we need to write a
 * new TSUID entry or not. It will not attempt to verify if the stored data is
 * valid, just checks to see if something is stored in the proper column.
 * @param tsdb The TSDB to use for storage access
 * @param tsuid The UID of the meta to verify
 * @return True if data was found, false if not
 * @throws HBaseException if there was an issue fetching
 */
public static Deferred<Boolean> counterExistsInStorage(final TSDB tsdb, final byte[] tsuid) {
    final GetRequest get = new GetRequest(tsdb.metaTable(), tsuid);
    get.family(FAMILY);
    get.qualifier(COUNTER_QUALIFIER);
    /**
     * Callback from the GetRequest that simply determines if the row is empty
     * or not
     */
    final class ExistsCB implements Callback<Boolean, ArrayList<KeyValue>> {

        @Override
        public Boolean call(ArrayList<KeyValue> row) throws Exception {
            if (row == null || row.isEmpty() || row.get(0).value() == null) {
                return false;
            }
            return true;
        }
    }
    return tsdb.getClient().get(get).addCallback(new ExistsCB());
}