Examples with ByteSet net.opentsdb.utils.ByteSet used on opensource projects

Example 6 with ByteSet

use of net.opentsdb.utils.ByteSet in project opentsdb by OpenTSDB.

the class TestHistogramDataPointsToDataPointsAdaptor method getTagUidsAggedNotInQuery.

@Test
public void getTagUidsAggedNotInQuery() throws Exception {
    final ByteSet query_tags = new ByteSet();
    query_tags.add(new byte[] { 0, 0, 0, 3 });
    final ByteMap<byte[]> uids = new ByteMap<byte[]>();
    uids.put(new byte[] { 0, 0, 0, 1 }, new byte[] { 0, 0, 0, 2 });
    final HistogramSpan span = mock(HistogramSpan.class);
    when(span.getTagUids()).thenReturn(uids);
    DownsamplingSpecification specification = new DownsamplingSpecification("1dc-sum");
    final HistogramSpanGroup group = PowerMockito.spy(new HistogramSpanGroup(tsdb, start_ts, end_ts, null, HistogramAggregation.SUM, specification, 0, 0, 0, false, query_tags));
    final ArrayList<HistogramSpan> spans = Whitebox.getInternalState(group, "spans");
    spans.add(span);
    HistogramDataPointsToDataPointsAdaptor dps_ada = new HistogramDataPointsToDataPointsAdaptor(group, 0.98f);
    final ByteMap<byte[]> uids_read = dps_ada.getTagUids();
    assertEquals(0, uids_read.size());
    final List<byte[]> agg_tags = dps_ada.getAggregatedTagUids();
    assertEquals(1, agg_tags.size());
    assertArrayEquals(new byte[] { 0, 0, 0, 1 }, agg_tags.get(0));
}

Example 7 with ByteSet

use of net.opentsdb.utils.ByteSet in project opentsdb by OpenTSDB.

the class TestIntersectionIterator method beforeLocal.

@Before
public void beforeLocal() throws Exception {
    tags = new ByteMap<byte[]>();
    tags.put(UID1, UID1);
    tags.put(UID2, UID2);
    agg_tags = new ByteSet();
    agg_tags.add(UID3);
    sub = mock(ITimeSyncedIterator.class);
    query_tags = new ByteSet();
    query_tags.add(UID1);
    when(sub.getQueryTagKs()).thenReturn(query_tags);
}

Example 8 with ByteSet

use of net.opentsdb.utils.ByteSet in project opentsdb by OpenTSDB.

the class MultiGetQuery method prepareConcurrentMultiGetTasks.

/**
 * Compiles the list of TSUIDs and GetRequests to send to execute against
 * storage. Each batch will only have requests for one salt, i.e a batch
 * will not have requests with multiple salts.
 */
@VisibleForTesting
void prepareConcurrentMultiGetTasks() {
    multi_get_wait_cnt = 0;
    prepare_multi_get_start_time = DateTime.currentTimeMillis();
    final List<Long> row_base_time_list;
    if (RollupQuery.isValidQuery(rollup_query)) {
        row_base_time_list = prepareRowBaseTimesRollup();
    } else {
        row_base_time_list = prepareRowBaseTimes();
    }
    int next_concurrency_index = 0;
    List<GetRequest> gets_to_prepare = new ArrayList<GetRequest>(batch_size);
    Set<byte[]> tsuids = new ByteSet();
    final ByteMap<ByteMap<List<GetRequest>>> all_tsuids_gets;
    if (multiget_no_meta) {
        // prepare the tagvs combinations and base time list
        final List<byte[][]> tagv_compinations = prepareAllTagvCompounds();
        all_tsuids_gets = prepareRequestsNoMeta(tagv_compinations, row_base_time_list);
    } else {
        all_tsuids_gets = prepareRequests(row_base_time_list, tags);
    }
    // Iterate over all salts
    for (final Entry<byte[], ByteMap<List<GetRequest>>> salts_entry : all_tsuids_gets.entrySet()) {
        if (gets_to_prepare.size() > 0) {
            // if we have any gets_to_prepare for previous salt, create a
            // request out of it.
            final MultiGetTask task = new MultiGetTask(tsuids, gets_to_prepare);
            final List<MultiGetTask> mulget_task_list = multi_get_tasks.get((next_concurrency_index++) % concurrency_multi_get);
            mulget_task_list.add(task);
            ++multi_get_wait_cnt;
            multi_get_num_get_requests = multi_get_num_get_requests + gets_to_prepare.size();
            gets_to_prepare = new ArrayList<GetRequest>(batch_size);
            tsuids = new ByteSet();
        }
        byte[] curr_salt = salts_entry.getKey();
        // Iterate over all tsuid's in curr_salt and add them to gets_to_prepare
        for (final Entry<byte[], List<GetRequest>> gets_entry : salts_entry.getValue()) {
            byte[] tsuid = gets_entry.getKey();
            for (GetRequest request : gets_entry.getValue()) {
                if (gets_to_prepare.size() >= batch_size) {
                    // close batch and create a MultiGetTask
                    final MultiGetTask task = new MultiGetTask(tsuids, gets_to_prepare);
                    final List<MultiGetTask> mulget_task_list = multi_get_tasks.get((next_concurrency_index++) % concurrency_multi_get);
                    mulget_task_list.add(task);
                    ++multi_get_wait_cnt;
                    multi_get_num_get_requests = multi_get_num_get_requests + gets_to_prepare.size();
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Finished preparing MultiGetRequest with " + gets_to_prepare.size() + " requests for salt " + curr_salt + " for tsuid " + Bytes.pretty(tsuid));
                    }
                    // prepare a new task list and tsuids
                    gets_to_prepare = new ArrayList<GetRequest>(batch_size);
                    tsuids = new ByteSet();
                }
                gets_to_prepare.add(request);
                tsuids.add(gets_entry.getKey());
            }
            tsuids.add(gets_entry.getKey());
        }
    }
    if (gets_to_prepare.size() > 0) {
        final MultiGetTask task = new MultiGetTask(tsuids, gets_to_prepare);
        final List<MultiGetTask> mulget_task_list = multi_get_tasks.get((next_concurrency_index++) % concurrency_multi_get);
        mulget_task_list.add(task);
        ++multi_get_wait_cnt;
        multi_get_num_get_requests = multi_get_num_get_requests + gets_to_prepare.size();
        LOG.debug("Finished preparing MultiGetRequest with " + gets_to_prepare.size());
        gets_to_prepare = new ArrayList<GetRequest>(batch_size);
        tsuids = new ByteSet();
    }
    prepare_multi_get_end_time = DateTime.currentTimeMillis();
    if (LOG.isDebugEnabled()) {
        LOG.debug("Finished preparing concurrency multi get task with " + multi_get_wait_cnt + " tasks using " + (prepare_multi_get_end_time - prepare_multi_get_start_time) + "ms");
    }
}

Example 9 with ByteSet

use of net.opentsdb.utils.ByteSet in project opentsdb by OpenTSDB.

the class TestHistogramDataPointsToDataPointsAdaptor method iteratorAllItemsWithDiffPercentile.

@Test
public void iteratorAllItemsWithDiffPercentile() {
    List<HistogramDataPoint> row = new ArrayList<HistogramDataPoint>();
    for (int i = 0; i < 10; ++i) {
        row.add(new SimpleHistogramDataPointAdapter(new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i));
    }
    final HistogramSpan hspan = new HistogramSpan(tsdb);
    hspan.addRow(KEY, row);
    List<HistogramSpan> spans = new ArrayList<HistogramSpan>();
    spans.add(hspan);
    final ByteSet query_tags = new ByteSet();
    query_tags.add(new byte[] { 0, 0, 0, 1 });
    HistogramSpanGroup hist_span_group = new HistogramSpanGroup(tsdb, BASE_TIME, BASE_TIME + 5000L * 10, spans, HistogramAggregation.SUM, DownsamplingSpecification.NO_DOWNSAMPLER, 0, 0, 0, false, query_tags);
    // 98 percentile
    HistogramDataPointsToDataPointsAdaptor dps_ada_98 = new HistogramDataPointsToDataPointsAdaptor(hist_span_group, 0.98f);
    List<Double> values = new ArrayList<Double>();
    for (DataPoint dp : dps_ada_98) {
        assertFalse(dp.isInteger());
        values.add(dp.doubleValue());
    }
    // end for
    assertTrue(dps_ada_98.isPercentile());
    List<Double> to_checks = new ArrayList<Double>();
    for (int i = 0; i < 10; ++i) {
        to_checks.add(i * 0.98);
    }
    // end for
    assertEquals(values.size(), to_checks.size());
    for (int i = 0; i < values.size(); ++i) {
        assertEquals(values.get(i).doubleValue(), to_checks.get(i).doubleValue(), 0.0001);
    }
    // end for
    // 95 percentile
    HistogramDataPointsToDataPointsAdaptor dps_ada_95 = new HistogramDataPointsToDataPointsAdaptor(hist_span_group, 0.95f);
    List<Double> values_95 = new ArrayList<Double>();
    for (DataPoint dp : dps_ada_95) {
        assertFalse(dp.isInteger());
        values_95.add(dp.doubleValue());
    }
    // end for
    assertTrue(dps_ada_95.isPercentile());
    List<Double> to_checks_95 = new ArrayList<Double>();
    for (int i = 0; i < 10; ++i) {
        to_checks_95.add(i * 0.95);
    }
    // end for
    assertEquals(values_95.size(), to_checks_95.size());
    for (int i = 0; i < values.size(); ++i) {
        assertEquals(values_95.get(i).doubleValue(), to_checks_95.get(i).doubleValue(), 0.0001);
    }
// end for
}

Example 10 with ByteSet

use of net.opentsdb.utils.ByteSet in project opentsdb by OpenTSDB.

the class TestHistogramDataPointsToDataPointsAdaptor method iteratorAllItems.

@Test
public void iteratorAllItems() {
    List<HistogramDataPoint> row = new ArrayList<HistogramDataPoint>();
    for (int i = 0; i < 10; ++i) {
        row.add(new SimpleHistogramDataPointAdapter(new LongHistogramDataPointForTest(0, i), BASE_TIME + 5000L * i));
    }
    final HistogramSpan hspan = new HistogramSpan(tsdb);
    hspan.addRow(KEY, row);
    List<HistogramSpan> spans = new ArrayList<HistogramSpan>();
    spans.add(hspan);
    final ByteSet query_tags = new ByteSet();
    query_tags.add(new byte[] { 0, 0, 0, 1 });
    HistogramSpanGroup hist_span_group = new HistogramSpanGroup(tsdb, BASE_TIME, BASE_TIME + 5000L * 10, spans, HistogramAggregation.SUM, DownsamplingSpecification.NO_DOWNSAMPLER, 0, 0, 0, false, query_tags);
    HistogramDataPointsToDataPointsAdaptor dps_ada = new HistogramDataPointsToDataPointsAdaptor(hist_span_group, 0.98f);
    List<Double> values = new ArrayList<Double>();
    List<Long> timestamp_in_ms = new ArrayList<Long>();
    for (DataPoint dp : dps_ada) {
        assertFalse(dp.isInteger());
        values.add(dp.doubleValue());
        timestamp_in_ms.add(dp.timestamp());
    }
    // end for
    assertTrue(dps_ada.isPercentile());
    List<Double> to_checks = new ArrayList<Double>();
    for (int i = 0; i < 10; ++i) {
        to_checks.add(i * 0.98);
    }
    // end for
    assertEquals(values.size(), to_checks.size());
    for (int i = 0; i < values.size(); ++i) {
        assertEquals(values.get(i).doubleValue(), to_checks.get(i).doubleValue(), 0.0001);
        assertEquals(timestamp_in_ms.get(i).longValue(), BASE_TIME + 5000L * i);
    }
// end for
}