Examples with MultiGetTask net.opentsdb.core.MultiGetQuery.MultiGetTask used on opensource projects

Example 1 with MultiGetTask

use of net.opentsdb.core.MultiGetQuery.MultiGetTask in project opentsdb by OpenTSDB.

the class TestMultiGetQuery method prepareConcurrentMultiGetTasksSmallBatchAndSmallConcurrent.

@Test
public void prepareConcurrentMultiGetTasksSmallBatchAndSmallConcurrent() throws Exception {
    config.overrideConfig("tsd.query.multi_get.concurrent", "2");
    config.overrideConfig("tsd.query.multi_get.batch_size", "17");
    MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, 0, max_bytes, false, multiget_no_meta);
    mgq.prepareConcurrentMultiGetTasks();
    final List<List<MultiGetTask>> tasks = mgq.getMultiGetTasks();
    assertEquals(config.getInt("tsd.query.multi_get.concurrent"), tasks.size());
    assertEquals(2, tasks.get(0).size());
    assertEquals(1, tasks.get(1).size());
    for (List<MultiGetTask> taskList : tasks) {
        for (MultiGetTask task : taskList) {
            byte salt = task.getGets().get(0).key()[0];
            for (GetRequest request : task.getGets()) {
                assertEquals(salt, request.key()[0]);
            }
        }
    }
    // first batch
    MultiGetTask task = tasks.get(0).get(0);
    assertEquals(1, task.getTSUIDs().size());
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "A", TAGK_B_STRING, "D")));
    assertEquals(17, task.getGets().size());
    int idx = 0;
    int ts = 1481227200;
    while (idx < 16) {
        // notice the early cut off. The last hour should spill over.
        assertArrayEquals(task.getGets().get(idx++).key(), getRowKey(METRIC_STRING, ts, TAGK_STRING, "A", TAGK_B_STRING, "D"));
        ts += 3600;
    }
    // 17th request
    assertArrayEquals(task.getGets().get(idx++).key(), getRowKey(METRIC_STRING, ts, TAGK_STRING, "A", TAGK_B_STRING, "D"));
    ts += 3600;
    // next batch
    task = tasks.get(1).get(0);
    assertEquals(1, task.getTSUIDs().size());
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "B", TAGK_B_STRING, "D")));
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "B", TAGK_B_STRING, "D")));
    assertEquals(17, task.getGets().size());
    idx = 0;
    ts = 1481227200;
    while (idx < 17) {
        assertArrayEquals(task.getGets().get(idx++).key(), getRowKey(METRIC_STRING, ts, TAGK_STRING, "B", TAGK_B_STRING, "D"));
        ts += 3600;
    }
    // next batch
    task = tasks.get(0).get(1);
    assertEquals(1, task.getTSUIDs().size());
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "C", TAGK_B_STRING, "D")));
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "C", TAGK_B_STRING, "D")));
    assertEquals(17, task.getGets().size());
    idx = 0;
    ts = 1481227200;
    while (idx < 17) {
        assertArrayEquals(task.getGets().get(idx++).key(), getRowKey(METRIC_STRING, ts, TAGK_STRING, "C", TAGK_B_STRING, "D"));
        ts += 3600;
    }
}

Example 2 with MultiGetTask

use of net.opentsdb.core.MultiGetQuery.MultiGetTask in project opentsdb by OpenTSDB.

the class TestMultiGetQuery method prepareConcurrentMultiGetTasks.

@Test
public void prepareConcurrentMultiGetTasks() throws Exception {
    MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, 0, max_bytes, false, multiget_no_meta);
    mgq.prepareConcurrentMultiGetTasks();
    final List<List<MultiGetTask>> tasks = mgq.getMultiGetTasks();
    assertEquals(config.getInt("tsd.query.multi_get.concurrent"), tasks.size());
    for (int i = 1; i < tasks.size(); i++) {
        assertTrue(tasks.get(i).isEmpty());
    }
    for (List<MultiGetTask> taskList : tasks) {
        for (MultiGetTask task : taskList) {
            byte salt = task.getGets().get(0).key()[0];
            for (GetRequest request : task.getGets()) {
                assertEquals(salt, request.key()[0]);
            }
        }
    }
    assertEquals(1, tasks.get(0).size());
    MultiGetTask task = tasks.get(0).get(0);
    assertEquals(3, task.getTSUIDs().size());
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "A", TAGK_B_STRING, "D")));
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "B", TAGK_B_STRING, "D")));
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "C", TAGK_B_STRING, "D")));
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "A", TAGK_B_STRING, "D")));
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "B", TAGK_B_STRING, "D")));
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "C", TAGK_B_STRING, "D")));
    assertEquals(51, task.getGets().size());
    // 6 sets of 17 timestamps. Ugly UT
    int idx = 0;
    int ts = 1481227200;
    while (idx < 17) {
        assertArrayEquals(task.getGets().get(idx++).key(), getRowKey(METRIC_STRING, ts, TAGK_STRING, "A", TAGK_B_STRING, "D"));
        ts += 3600;
    }
    ts = 1481227200;
    while (idx < 17) {
        assertArrayEquals(task.getGets().get(idx++).key(), getRowKey(METRIC_STRING, ts, TAGK_STRING, "B", TAGK_B_STRING, "D"));
        ts += 3600;
    }
    ts = 1481227200;
    while (idx < 17) {
        assertArrayEquals(task.getGets().get(idx++).key(), getRowKey(METRIC_STRING, ts, TAGK_STRING, "C", TAGK_B_STRING, "D"));
        ts += 3600;
    }
}

Example 3 with MultiGetTask

use of net.opentsdb.core.MultiGetQuery.MultiGetTask in project opentsdb by OpenTSDB.

the class TestMultiGetQuery method prepareConcurrentMultiGetTasksSmallBatch.

@Test
public void prepareConcurrentMultiGetTasksSmallBatch() throws Exception {
    config.overrideConfig("tsd.query.multi_get.batch_size", "17");
    MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, 0, max_bytes, false, multiget_no_meta);
    mgq.prepareConcurrentMultiGetTasks();
    final List<List<MultiGetTask>> tasks = mgq.getMultiGetTasks();
    assertEquals(config.getInt("tsd.query.multi_get.concurrent"), tasks.size());
    assertEquals(1, tasks.get(0).size());
    // first batch
    MultiGetTask task = tasks.get(0).get(0);
    Set<byte[]> tsuids = task.getTSUIDs();
    assertEquals(1, task.getTSUIDs().size());
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "A", TAGK_B_STRING, "D")));
    assertEquals(17, task.getGets().size());
    int idx = 0;
    int ts = 1481227200;
    while (idx < 17) {
        // notice the early cut off. The last hour should spill over.
        assertArrayEquals(task.getGets().get(idx++).key(), getRowKey(METRIC_STRING, ts, TAGK_STRING, "A", TAGK_B_STRING, "D"));
        ts += 3600;
    }
    // next batch
    task = tasks.get(1).get(0);
    assertEquals(1, task.getTSUIDs().size());
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "B", TAGK_B_STRING, "D")));
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "B", TAGK_B_STRING, "D")));
    assertEquals(17, task.getGets().size());
    idx = 0;
    ts = 1481227200;
    while (idx < 17) {
        assertArrayEquals(task.getGets().get(idx++).key(), getRowKey(METRIC_STRING, ts, TAGK_STRING, "B", TAGK_B_STRING, "D"));
        ts += 3600;
    }
    // next batch
    task = tasks.get(2).get(0);
    assertEquals(1, task.getTSUIDs().size());
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "C", TAGK_B_STRING, "D")));
    assertNotNull(task.getTSUIDs().contains(getTSUID(METRIC_STRING, TAGK_STRING, "C", TAGK_B_STRING, "D")));
    assertEquals(17, task.getGets().size());
    idx = 0;
    ts = 1481227200;
    while (idx < 17) {
        assertArrayEquals(task.getGets().get(idx++).key(), getRowKey(METRIC_STRING, ts, TAGK_STRING, "C", TAGK_B_STRING, "D"));
        ts += 3600;
    }
}

Example 4 with MultiGetTask

use of net.opentsdb.core.MultiGetQuery.MultiGetTask in project opentsdb by OpenTSDB.

the class TestMultiGetQuery method prepareConcurrentMultiSortedSalts.

@Test
public void prepareConcurrentMultiSortedSalts() throws Exception {
    config.overrideConfig("tsd.query.multi_get.concurrent", "2");
    config.overrideConfig("tsd.query.multi_get.batch_size", "2");
    Const.setSaltWidth(1);
    MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, 0, max_bytes, false, multiget_no_meta);
    mgq.prepareConcurrentMultiGetTasks();
    final List<List<MultiGetTask>> tasks = mgq.getMultiGetTasks();
    assertEquals(config.getInt("tsd.query.multi_get.concurrent"), tasks.size());
    for (List<MultiGetTask> taskList : tasks) {
        for (MultiGetTask task : taskList) {
            byte salt = task.getGets().get(0).key()[0];
            for (GetRequest request : task.getGets()) {
                assertEquals(salt, request.key()[0]);
            }
        }
    }
    Const.setSaltWidth(0);
}