Examples with DimensionSelector org.apache.druid.segment.DimensionSelector used on opensource projects

Example 51 with DimensionSelector

use of org.apache.druid.segment.DimensionSelector in project druid by druid-io.

the class IncrementalIndexStorageAdapterTest method testCursoringAndIndexUpdationInterleaving.

@Test
public void testCursoringAndIndexUpdationInterleaving() throws Exception {
    final IncrementalIndex index = indexCreator.createIndex();
    final long timestamp = System.currentTimeMillis();
    for (int i = 0; i < 2; i++) {
        index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v1" + i)));
    }
    final StorageAdapter sa = new IncrementalIndexStorageAdapter(index);
    Sequence<Cursor> cursors = sa.makeCursors(null, Intervals.utc(timestamp - 60_000, timestamp + 60_000), VirtualColumns.EMPTY, Granularities.ALL, false, null);
    final AtomicInteger assertCursorsNotEmpty = new AtomicInteger(0);
    cursors.map(cursor -> {
        DimensionSelector dimSelector = cursor.getColumnSelectorFactory().makeDimensionSelector(new DefaultDimensionSpec("billy", "billy"));
        int cardinality = dimSelector.getValueCardinality();
        // index gets more rows at this point, while other thread is iterating over the cursor
        try {
            for (int i = 0; i < 1; i++) {
                index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v2" + i)));
            }
        } catch (Exception ex) {
            throw new RuntimeException(ex);
        }
        int rowNumInCursor = 0;
        // and then, cursoring continues in the other thread
        while (!cursor.isDone()) {
            IndexedInts row = dimSelector.getRow();
            row.forEach(i -> Assert.assertTrue(i < cardinality));
            cursor.advance();
            rowNumInCursor++;
        }
        Assert.assertEquals(2, rowNumInCursor);
        assertCursorsNotEmpty.incrementAndGet();
        return null;
    }).toList();
    Assert.assertEquals(1, assertCursorsNotEmpty.get());
}

Example 52 with DimensionSelector

use of org.apache.druid.segment.DimensionSelector in project druid by druid-io.

the class IncrementalIndexStorageAdapterTest method testCursoringAndSnapshot.

@Test
public void testCursoringAndSnapshot() throws Exception {
    final IncrementalIndex index = indexCreator.createIndex();
    final long timestamp = System.currentTimeMillis();
    for (int i = 0; i < 2; i++) {
        index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v0" + i)));
    }
    final StorageAdapter sa = new IncrementalIndexStorageAdapter(index);
    Sequence<Cursor> cursors = sa.makeCursors(null, Intervals.utc(timestamp - 60_000, timestamp + 60_000), VirtualColumns.EMPTY, Granularities.ALL, false, null);
    final AtomicInteger assertCursorsNotEmpty = new AtomicInteger(0);
    cursors.map(cursor -> {
        DimensionSelector dimSelector1A = cursor.getColumnSelectorFactory().makeDimensionSelector(new DefaultDimensionSpec("billy", "billy"));
        int cardinalityA = dimSelector1A.getValueCardinality();
        // index gets more rows at this point, while other thread is iterating over the cursor
        try {
            index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v1")));
        } catch (Exception ex) {
            throw new RuntimeException(ex);
        }
        DimensionSelector dimSelector1B = cursor.getColumnSelectorFactory().makeDimensionSelector(new DefaultDimensionSpec("billy", "billy"));
        // index gets more rows at this point, while other thread is iterating over the cursor
        try {
            index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v2")));
            index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy2"), ImmutableMap.of("billy2", "v3")));
        } catch (Exception ex) {
            throw new RuntimeException(ex);
        }
        DimensionSelector dimSelector1C = cursor.getColumnSelectorFactory().makeDimensionSelector(new DefaultDimensionSpec("billy", "billy"));
        DimensionSelector dimSelector2D = cursor.getColumnSelectorFactory().makeDimensionSelector(new DefaultDimensionSpec("billy2", "billy2"));
        // index gets more rows at this point, while other thread is iterating over the cursor
        try {
            index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy"), ImmutableMap.of("billy", "v3")));
            index.add(new MapBasedInputRow(timestamp, Collections.singletonList("billy3"), ImmutableMap.of("billy3", "")));
        } catch (Exception ex) {
            throw new RuntimeException(ex);
        }
        DimensionSelector dimSelector3E = cursor.getColumnSelectorFactory().makeDimensionSelector(new DefaultDimensionSpec("billy3", "billy3"));
        int rowNumInCursor = 0;
        // and then, cursoring continues in the other thread
        while (!cursor.isDone()) {
            IndexedInts rowA = dimSelector1A.getRow();
            rowA.forEach(i -> Assert.assertTrue(i < cardinalityA));
            IndexedInts rowB = dimSelector1B.getRow();
            rowB.forEach(i -> Assert.assertTrue(i < cardinalityA));
            IndexedInts rowC = dimSelector1C.getRow();
            rowC.forEach(i -> Assert.assertTrue(i < cardinalityA));
            IndexedInts rowD = dimSelector2D.getRow();
            // no null id, so should get empty dims array
            Assert.assertEquals(0, rowD.size());
            IndexedInts rowE = dimSelector3E.getRow();
            if (NullHandling.replaceWithDefault()) {
                Assert.assertEquals(1, rowE.size());
                // the null id
                Assert.assertEquals(0, rowE.get(0));
            } else {
                Assert.assertEquals(0, rowE.size());
            }
            cursor.advance();
            rowNumInCursor++;
        }
        Assert.assertEquals(2, rowNumInCursor);
        assertCursorsNotEmpty.incrementAndGet();
        return null;
    }).toList();
    Assert.assertEquals(1, assertCursorsNotEmpty.get());
}

Example 53 with DimensionSelector

use of org.apache.druid.segment.DimensionSelector in project druid by druid-io.

the class BloomFilterAggregatorTest method testBufferAggregateStringValues.

@Test
public void testBufferAggregateStringValues() throws IOException {
    DimensionSelector dimSelector = new CardinalityAggregatorTest.TestDimensionSelector(VALUES2, null);
    StringBloomFilterAggregator agg = new StringBloomFilterAggregator(dimSelector, MAX_NUM_VALUES, true);
    int maxSize = valueAggregatorFactory.getMaxIntermediateSizeWithNulls();
    ByteBuffer buf = ByteBuffer.allocate(maxSize + 64);
    int pos = 10;
    buf.limit(pos + maxSize);
    agg.init(buf, pos);
    for (int i = 0; i < VALUES2.size(); ++i) {
        bufferAggregateDimension(Collections.singletonList(dimSelector), agg, buf, pos);
    }
    BloomKFilter bloomKFilter = BloomKFilter.deserialize((ByteBuffer) valueAggregatorFactory.finalizeComputation(agg.get(buf, pos)));
    String serialized = filterToString(bloomKFilter);
    Assert.assertEquals(serializedFilter2, serialized);
}

Example 54 with DimensionSelector

use of org.apache.druid.segment.DimensionSelector in project druid by druid-io.

the class PooledTopNAlgorithm method scanAndAggregateDefault.

/**
 * Use aggressive loop unrolling to aggregate the data
 *
 * How this works: The aggregates are evaluated AGG_UNROLL_COUNT at a time. This was chosen to be 8 rather arbitrarily.
 * The offsets into the output buffer are precalculated and stored in aggregatorOffsets
 *
 * For queries whose aggregate count is less than AGG_UNROLL_COUNT, the aggregates evaluted in a switch statement.
 * See http://en.wikipedia.org/wiki/Duff's_device for more information on this kind of approach
 *
 * This allows out of order execution of the code. In local tests, the JVM inlines all the way to this function.
 *
 * If there are more than AGG_UNROLL_COUNT aggregates, then the remainder is calculated with the switch, and the
 * blocks of AGG_UNROLL_COUNT are calculated in a partially unrolled for-loop.
 *
 * Putting the switch first allows for optimization for the common case (less than AGG_UNROLL_COUNT aggs) but
 * still optimizes the high quantity of aggregate queries which benefit greatly from any speed improvements
 * (they simply take longer to start with).
 */
private static long scanAndAggregateDefault(final PooledTopNParams params, final int[] positions, final BufferAggregator[] theAggregators) {
    if (params.getCardinality() < 0) {
        throw new UnsupportedOperationException("Cannot operate on a dimension with unknown cardinality");
    }
    final ByteBuffer resultsBuf = params.getResultsBuf();
    final int numBytesPerRecord = params.getNumBytesPerRecord();
    final int[] aggregatorSizes = params.getAggregatorSizes();
    final Cursor cursor = params.getCursor();
    final DimensionSelector dimSelector = params.getDimSelector();
    final int[] aggregatorOffsets = new int[aggregatorSizes.length];
    for (int j = 0, offset = 0; j < aggregatorSizes.length; ++j) {
        aggregatorOffsets[j] = offset;
        offset += aggregatorSizes[j];
    }
    final int aggSize = theAggregators.length;
    final int aggExtra = aggSize % AGG_UNROLL_COUNT;
    int currentPosition = 0;
    long processedRows = 0;
    while (!cursor.isDoneOrInterrupted()) {
        final IndexedInts dimValues = dimSelector.getRow();
        final int dimSize = dimValues.size();
        final int dimExtra = dimSize % AGG_UNROLL_COUNT;
        switch(dimExtra) {
            case 7:
                currentPosition = aggregateDimValue(positions, theAggregators, resultsBuf, numBytesPerRecord, aggregatorOffsets, aggSize, aggExtra, dimValues.get(6), currentPosition);
            // fall through
            case 6:
                currentPosition = aggregateDimValue(positions, theAggregators, resultsBuf, numBytesPerRecord, aggregatorOffsets, aggSize, aggExtra, dimValues.get(5), currentPosition);
            // fall through
            case 5:
                currentPosition = aggregateDimValue(positions, theAggregators, resultsBuf, numBytesPerRecord, aggregatorOffsets, aggSize, aggExtra, dimValues.get(4), currentPosition);
            // fall through
            case 4:
                currentPosition = aggregateDimValue(positions, theAggregators, resultsBuf, numBytesPerRecord, aggregatorOffsets, aggSize, aggExtra, dimValues.get(3), currentPosition);
            // fall through
            case 3:
                currentPosition = aggregateDimValue(positions, theAggregators, resultsBuf, numBytesPerRecord, aggregatorOffsets, aggSize, aggExtra, dimValues.get(2), currentPosition);
            // fall through
            case 2:
                currentPosition = aggregateDimValue(positions, theAggregators, resultsBuf, numBytesPerRecord, aggregatorOffsets, aggSize, aggExtra, dimValues.get(1), currentPosition);
            // fall through
            case 1:
                currentPosition = aggregateDimValue(positions, theAggregators, resultsBuf, numBytesPerRecord, aggregatorOffsets, aggSize, aggExtra, dimValues.get(0), currentPosition);
        }
        for (int i = dimExtra; i < dimSize; i += AGG_UNROLL_COUNT) {
            currentPosition = aggregateDimValue(positions, theAggregators, resultsBuf, numBytesPerRecord, aggregatorOffsets, aggSize, aggExtra, dimValues.get(i), currentPosition);
            currentPosition = aggregateDimValue(positions, theAggregators, resultsBuf, numBytesPerRecord, aggregatorOffsets, aggSize, aggExtra, dimValues.get(i + 1), currentPosition);
            currentPosition = aggregateDimValue(positions, theAggregators, resultsBuf, numBytesPerRecord, aggregatorOffsets, aggSize, aggExtra, dimValues.get(i + 2), currentPosition);
            currentPosition = aggregateDimValue(positions, theAggregators, resultsBuf, numBytesPerRecord, aggregatorOffsets, aggSize, aggExtra, dimValues.get(i + 3), currentPosition);
            currentPosition = aggregateDimValue(positions, theAggregators, resultsBuf, numBytesPerRecord, aggregatorOffsets, aggSize, aggExtra, dimValues.get(i + 4), currentPosition);
            currentPosition = aggregateDimValue(positions, theAggregators, resultsBuf, numBytesPerRecord, aggregatorOffsets, aggSize, aggExtra, dimValues.get(i + 5), currentPosition);
            currentPosition = aggregateDimValue(positions, theAggregators, resultsBuf, numBytesPerRecord, aggregatorOffsets, aggSize, aggExtra, dimValues.get(i + 6), currentPosition);
            currentPosition = aggregateDimValue(positions, theAggregators, resultsBuf, numBytesPerRecord, aggregatorOffsets, aggSize, aggExtra, dimValues.get(i + 7), currentPosition);
        }
        cursor.advanceUninterruptibly();
        processedRows++;
    }
    return processedRows;
}

Example 55 with DimensionSelector

use of org.apache.druid.segment.DimensionSelector in project druid by druid-io.

the class PooledTopNAlgorithm method updateResults.

@Override
protected void updateResults(PooledTopNParams params, int[] positions, BufferAggregator[] theAggregators, TopNResultBuilder resultBuilder) {
    if (params.getCardinality() < 0) {
        throw new UnsupportedOperationException("Cannot operate on a dimension with unknown cardinality");
    }
    final ByteBuffer resultsBuf = params.getResultsBuf();
    final int[] aggregatorSizes = params.getAggregatorSizes();
    final DimensionSelector dimSelector = params.getDimSelector();
    for (int i = 0; i < positions.length; i++) {
        int position = positions[i];
        if (position >= 0) {
            Object[] vals = new Object[theAggregators.length];
            for (int j = 0; j < theAggregators.length; j++) {
                vals[j] = theAggregators[j].get(resultsBuf, position);
                position += aggregatorSizes[j];
            }
            // Output type must be STRING in order for PooledTopNAlgorithm to make sense; so no need to convert value.
            resultBuilder.addEntry(dimSelector.lookupName(i), i, vals);
        }
    }
}