Examples with ByteSet net.opentsdb.utils.ByteSet used on opensource projects

Example 1 with ByteSet

use of net.opentsdb.utils.ByteSet in project opentsdb by OpenTSDB.

the class IntersectionIterator method flattenTags.

/**
 * Flattens the appropriate tags into a single byte array
 * @param use_query_tags Whether or not to include tags returned with the
 * results or just use those group by'd in the query
 * @param include_agg_tags Whether or not to include the aggregated tags in
 * the identifier
 * @param tags The map of tags from the result set
 * @param agg_tags The list of aggregated tags
 * @param sub The sub query iterator
 * @return A byte array with the flattened tag keys and values. Note that
 * if the tags set is empty, this may return an empty array (but not a null
 * array)
 */
static byte[] flattenTags(final boolean use_query_tags, final boolean include_agg_tags, final ByteMap<byte[]> tags, final ByteSet agg_tags, final ITimeSyncedIterator sub) {
    if (tags.isEmpty()) {
        return HBaseClient.EMPTY_ARRAY;
    }
    final ByteSet query_tagks;
    // NOTE: We MAY need the agg tags but I'm not sure yet
    final int tag_size;
    if (use_query_tags) {
        int i = 0;
        if (sub.getQueryTagKs() != null && !sub.getQueryTagKs().isEmpty()) {
            query_tagks = sub.getQueryTagKs();
            for (final Map.Entry<byte[], byte[]> pair : tags.entrySet()) {
                if (query_tagks.contains(pair.getKey())) {
                    i++;
                }
            }
        } else {
            query_tagks = new ByteSet();
        }
        tag_size = i;
    } else {
        query_tagks = new ByteSet();
        tag_size = tags.size();
    }
    int len = (tag_size * (TSDB.tagk_width() + TSDB.tagv_width())) + (include_agg_tags ? (agg_tags.size() * TSDB.tagk_width()) : 0);
    final byte[] tagks = new byte[len];
    int i = 0;
    for (final Map.Entry<byte[], byte[]> pair : tags.entrySet()) {
        if (use_query_tags && !query_tagks.contains(pair.getKey())) {
            continue;
        }
        System.arraycopy(pair.getKey(), 0, tagks, i, TSDB.tagk_width());
        i += TSDB.tagk_width();
        System.arraycopy(pair.getValue(), 0, tagks, i, TSDB.tagv_width());
        i += TSDB.tagv_width();
    }
    if (include_agg_tags) {
        for (final byte[] tagk : agg_tags) {
            System.arraycopy(tagk, 0, tagks, i, TSDB.tagk_width());
            i += TSDB.tagk_width();
        }
    }
    return tagks;
}

Example 2 with ByteSet

use of net.opentsdb.utils.ByteSet in project opentsdb by OpenTSDB.

the class TestMultiGetQuery method prepareRequests.

@Test
public void prepareRequests() throws Exception {
    MultiGetQuery mgq = new MultiGetQuery(tsdb, query, METRIC_BYTES, q_tags, start_ts, end_ts, tsdb.dataTable(), spans, null, 0, null, query_stats, 0, max_bytes, false, multiget_no_meta);
    List<Long> timestamps = mgq.prepareRowBaseTimes();
    ByteMap<ByteMap<List<GetRequest>>> row_map = mgq.prepareRequests(timestamps, q_tags);
    ByteSet tsuids = new ByteSet();
    for (ByteMap<List<GetRequest>> rows : row_map.values()) {
        tsuids.addAll(rows.keySet());
    }
    assertEquals(3, tsuids.size());
    List<GetRequest> rows = new ArrayList<GetRequest>();
    for (Entry<byte[], ByteMap<List<GetRequest>>> salt_entry : row_map.entrySet()) {
        System.out.println(salt_entry.getValue());
        rows.addAll(salt_entry.getValue().get(getTSUID(METRIC_STRING, TAGK_STRING, "A", TAGK_B_STRING, "D")));
    }
    assertEquals(timestamps.size(), rows.size());
    for (int i = 0; i < timestamps.size(); i++) {
        byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), TAGK_STRING, "A", TAGK_B_STRING, "D");
        assertArrayEquals(key, rows.get(i).key());
    }
    rows = new ArrayList<GetRequest>();
    for (Entry<byte[], ByteMap<List<GetRequest>>> salt_entry : row_map.entrySet()) {
        rows.addAll(salt_entry.getValue().get(getTSUID(METRIC_STRING, TAGK_STRING, "B", TAGK_B_STRING, "D")));
    }
    assertEquals(timestamps.size(), rows.size());
    for (int i = 0; i < timestamps.size(); i++) {
        byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), TAGK_STRING, "B", TAGK_B_STRING, "D");
        assertArrayEquals(key, rows.get(i).key());
    }
    rows = new ArrayList<GetRequest>();
    for (Entry<byte[], ByteMap<List<GetRequest>>> salt_entry : row_map.entrySet()) {
        rows.addAll(salt_entry.getValue().get(getTSUID(METRIC_STRING, TAGK_STRING, "C", TAGK_B_STRING, "D")));
    }
    assertEquals(timestamps.size(), rows.size());
    for (int i = 0; i < timestamps.size(); i++) {
        byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), TAGK_STRING, "C", TAGK_B_STRING, "D");
        assertArrayEquals(key, rows.get(i).key());
    }
    rows = new ArrayList<GetRequest>();
    for (Entry<byte[], ByteMap<List<GetRequest>>> salt_entry : row_map.entrySet()) {
        rows.addAll(salt_entry.getValue().get(getTSUID(METRIC_STRING, TAGK_STRING, "A", TAGK_B_STRING, "D")));
    }
    assertEquals(timestamps.size(), rows.size());
    for (int i = 0; i < timestamps.size(); i++) {
        byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), TAGK_STRING, "A", TAGK_B_STRING, "D");
        assertArrayEquals(key, rows.get(i).key());
    }
    rows = new ArrayList<GetRequest>();
    for (Entry<byte[], ByteMap<List<GetRequest>>> salt_entry : row_map.entrySet()) {
        rows.addAll(salt_entry.getValue().get(getTSUID(METRIC_STRING, TAGK_STRING, "B", TAGK_B_STRING, "D")));
    }
    assertEquals(timestamps.size(), rows.size());
    for (int i = 0; i < timestamps.size(); i++) {
        byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), TAGK_STRING, "B", TAGK_B_STRING, "D");
        assertArrayEquals(key, rows.get(i).key());
    }
    rows = new ArrayList<GetRequest>();
    for (Entry<byte[], ByteMap<List<GetRequest>>> salt_entry : row_map.entrySet()) {
        rows.addAll(salt_entry.getValue().get(getTSUID(METRIC_STRING, TAGK_STRING, "C", TAGK_B_STRING, "D")));
    }
    assertEquals(timestamps.size(), rows.size());
    for (int i = 0; i < timestamps.size(); i++) {
        byte[] key = getRowKey(METRIC_STRING, timestamps.get(i).intValue(), TAGK_STRING, "C", TAGK_B_STRING, "D");
        assertArrayEquals(key, rows.get(i).key());
    }
}

Example 3 with ByteSet

use of net.opentsdb.utils.ByteSet in project opentsdb by OpenTSDB.

the class TestHistogramSpanGroup method getTagUidsAggedNotInQuery.

@Test
public void getTagUidsAggedNotInQuery() throws Exception {
    final ByteSet query_tags = new ByteSet();
    query_tags.add(new byte[] { 0, 0, 0, 3 });
    final ByteMap<byte[]> uids = new ByteMap<byte[]>();
    uids.put(new byte[] { 0, 0, 0, 1 }, new byte[] { 0, 0, 0, 2 });
    final HistogramSpan span = mock(HistogramSpan.class);
    when(span.getTagUids()).thenReturn(uids);
    DownsamplingSpecification specification = new DownsamplingSpecification("1dc-sum");
    final HistogramSpanGroup group = PowerMockito.spy(new HistogramSpanGroup(tsdb, start_ts, end_ts, null, HistogramAggregation.SUM, specification, 0, 0, 0, false, query_tags));
    final ArrayList<HistogramSpan> spans = Whitebox.getInternalState(group, "spans");
    spans.add(span);
    final ByteMap<byte[]> uids_read = group.getTagUids();
    assertEquals(0, uids_read.size());
    final List<byte[]> agg_tags = group.getAggregatedTagUids();
    assertEquals(1, agg_tags.size());
    assertArrayEquals(new byte[] { 0, 0, 0, 1 }, agg_tags.get(0));
}

Example 4 with ByteSet

use of net.opentsdb.utils.ByteSet in project opentsdb by OpenTSDB.

the class TestHistogramSpanGroup method getTagUidsInQueryTags.

@Test
public void getTagUidsInQueryTags() throws Exception {
    final ByteSet query_tags = new ByteSet();
    query_tags.add(new byte[] { 0, 0, 0, 1 });
    final ByteMap<byte[]> uids = new ByteMap<byte[]>();
    uids.put(new byte[] { 0, 0, 0, 1 }, new byte[] { 0, 0, 0, 2 });
    final HistogramSpan span = mock(HistogramSpan.class);
    when(span.getTagUids()).thenReturn(uids);
    DownsamplingSpecification specification = new DownsamplingSpecification("1dc-sum");
    final HistogramSpanGroup group = PowerMockito.spy(new HistogramSpanGroup(tsdb, start_ts, end_ts, null, HistogramAggregation.SUM, specification, 0, 0, 0, false, query_tags));
    final ArrayList<HistogramSpan> spans = Whitebox.getInternalState(group, "spans");
    spans.add(span);
    final ByteMap<byte[]> uids_read = group.getTagUids();
    assertEquals(1, uids_read.size());
    assertEquals(0, Bytes.memcmp(new byte[] { 0, 0, 0, 1 }, uids_read.firstKey()));
    assertEquals(0, Bytes.memcmp(new byte[] { 0, 0, 0, 2 }, uids_read.firstEntry().getValue()));
    assertEquals(0, group.getAggregatedTagUids().size());
}

Example 5 with ByteSet

use of net.opentsdb.utils.ByteSet in project opentsdb by OpenTSDB.

the class TestHistogramDataPointsToDataPointsAdaptor method getTagUidsInQueryTags.

@Test
public void getTagUidsInQueryTags() throws Exception {
    final ByteSet query_tags = new ByteSet();
    query_tags.add(new byte[] { 0, 0, 0, 1 });
    final ByteMap<byte[]> uids = new ByteMap<byte[]>();
    uids.put(new byte[] { 0, 0, 0, 1 }, new byte[] { 0, 0, 0, 2 });
    final HistogramSpan span = mock(HistogramSpan.class);
    when(span.getTagUids()).thenReturn(uids);
    DownsamplingSpecification specification = new DownsamplingSpecification("1dc-sum");
    final HistogramSpanGroup group = PowerMockito.spy(new HistogramSpanGroup(tsdb, start_ts, end_ts, null, HistogramAggregation.SUM, specification, 0, 0, 0, false, query_tags));
    final ArrayList<HistogramSpan> spans = Whitebox.getInternalState(group, "spans");
    spans.add(span);
    HistogramDataPointsToDataPointsAdaptor dps_ada = new HistogramDataPointsToDataPointsAdaptor(group, 0.98f);
    final ByteMap<byte[]> uids_read = dps_ada.getTagUids();
    assertEquals(1, uids_read.size());
    assertEquals(0, Bytes.memcmp(new byte[] { 0, 0, 0, 1 }, uids_read.firstKey()));
    assertEquals(0, Bytes.memcmp(new byte[] { 0, 0, 0, 2 }, uids_read.firstEntry().getValue()));
    assertEquals(0, dps_ada.getAggregatedTagUids().size());
}