Search in sources :

Example 6 with ConciseBitmapFactory

use of org.apache.druid.collections.bitmap.ConciseBitmapFactory in project druid by druid-io.

the class LinearGutmanSplitStrategyTest method testPickSeeds.

@Test
public void testPickSeeds() {
    BitmapFactory bf = new ConciseBitmapFactory();
    LinearGutmanSplitStrategy strategy = new LinearGutmanSplitStrategy(0, 50, bf);
    Node node = new Node(new float[2], new float[2], true, bf);
    node.addChild(new Point(new float[] { 3, 7 }, 1, bf));
    node.addChild(new Point(new float[] { 1, 6 }, 1, bf));
    node.addChild(new Point(new float[] { 9, 8 }, 1, bf));
    node.addChild(new Point(new float[] { 2, 5 }, 1, bf));
    node.addChild(new Point(new float[] { 4, 4 }, 1, bf));
    node.enclose();
    Node[] groups = strategy.split(node);
    Assert.assertEquals(groups[0].getMinCoordinates()[0], 1.0f);
    Assert.assertEquals(groups[0].getMinCoordinates()[1], 4.0f);
    Assert.assertEquals(groups[1].getMinCoordinates()[0], 9.0f);
    Assert.assertEquals(groups[1].getMinCoordinates()[1], 8.0f);
}
Also used : ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory) Node(org.apache.druid.collections.spatial.Node) BitmapFactory(org.apache.druid.collections.bitmap.BitmapFactory) ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) Point(org.apache.druid.collections.spatial.Point) Test(org.junit.Test)

Example 7 with ConciseBitmapFactory

use of org.apache.druid.collections.bitmap.ConciseBitmapFactory in project druid by druid-io.

the class LinearGutmanSplitStrategyTest method testNumChildrenSize.

@Test
public void testNumChildrenSize() {
    BitmapFactory bf = new ConciseBitmapFactory();
    RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf);
    Random rand = ThreadLocalRandom.current();
    for (int i = 0; i < 100; i++) {
        tree.insert(new float[] { rand.nextFloat(), rand.nextFloat() }, i);
    }
    Assert.assertTrue(getNumPoints(tree.getRoot()) >= tree.getSize());
}
Also used : ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) Random(java.util.Random) BitmapFactory(org.apache.druid.collections.bitmap.BitmapFactory) ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) RTree(org.apache.druid.collections.spatial.RTree) Point(org.apache.druid.collections.spatial.Point) Test(org.junit.Test)

Example 8 with ConciseBitmapFactory

use of org.apache.druid.collections.bitmap.ConciseBitmapFactory in project druid by druid-io.

the class BitmapBenchmarkWithVaryingOrder method setup.

@Setup(Level.Trial)
public void setup() throws IOException {
    switch(type) {
        case "concise":
            bitmapFactory = new ConciseBitmapFactory();
            break;
        case "roaring":
            bitmapFactory = new RoaringBitmapFactory();
            break;
        default:
            throw new IAE("Unknown bitmap type[%s]", type);
    }
    bitmaps = new ArrayList<>(numBitmaps);
    // Bitmaps usually have a short circuit to early return an empty bitmap if it finds no intersection
    // during an AND operation. We want to let them iterate all bitmaps instead, so add some bits that
    // will be set for all bitmaps we create.
    final int[] knownTrue = new int[minIntersect];
    for (int i = 0; i < knownTrue.length; ++i) {
        knownTrue[i] = RANDOM.nextInt(bitmapLength);
    }
    for (int i = 0; i < numBitmaps; ++i) {
        // the later the bitmap is created, the higher its density is.
        final int bitCount = (int) (i * 0.1);
        IntSet ints = new IntOpenHashSet(bitCount);
        for (int j = 0; j < bitCount; j++) {
            int offset;
            do {
                offset = RANDOM.nextInt(bitmapLength);
            } while (ints.contains(offset));
            ints.add(offset);
        }
        final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
        ints.iterator().forEachRemaining((IntConsumer) mutableBitmap::add);
        for (int k : knownTrue) {
            mutableBitmap.add(k);
        }
        bitmaps.add(BitmapBenchmarkUtils.toOffheap(bitmapFactory.makeImmutableBitmap(mutableBitmap)));
    }
    reverseBitmaps = Lists.reverse(bitmaps);
}
Also used : IntOpenHashSet(it.unimi.dsi.fastutil.ints.IntOpenHashSet) ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory) IntSet(it.unimi.dsi.fastutil.ints.IntSet) MutableBitmap(org.apache.druid.collections.bitmap.MutableBitmap) IAE(org.apache.druid.java.util.common.IAE) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) Setup(org.openjdk.jmh.annotations.Setup)

Example 9 with ConciseBitmapFactory

use of org.apache.druid.collections.bitmap.ConciseBitmapFactory in project druid by druid-io.

the class UniformBitmapBenchmark method setup.

@Setup(Level.Trial)
public void setup() throws IOException {
    final int[] knownTrue = new int[minIntersect];
    for (int i = 0; i < knownTrue.length; ++i) {
        knownTrue[i] = RANDOM.nextInt(bitmapLength);
    }
    switch(type) {
        case "concise":
            bitmapFactory = new ConciseBitmapFactory();
            break;
        case "roaring":
            bitmapFactory = new RoaringBitmapFactory();
            break;
        default:
            throw new IAE("Unknown bitmap type[%s]", type);
    }
    bitmaps = new ArrayList<>(numBitmaps);
    for (int i = 0; i < numBitmaps; ++i) {
        final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
        for (int k = 0; k < bitmapLength; ++k) {
            if (RANDOM.nextDouble() < density) {
                mutableBitmap.add(k);
            }
        }
        for (int k : knownTrue) {
            mutableBitmap.add(k);
        }
        bitmaps.add(BitmapBenchmarkUtils.toOffheap(bitmapFactory.makeImmutableBitmap(mutableBitmap)));
    }
    final long totalSizeBytes = bitmaps.stream().mapToLong(bitmap -> bitmap.toBytes().length).sum();
    BitmapBenchmarkUtils.printSizeStats(type, density, bitmaps.size(), totalSizeBytes);
}
Also used : BenchmarkMode(org.openjdk.jmh.annotations.BenchmarkMode) Measurement(org.openjdk.jmh.annotations.Measurement) Blackhole(org.openjdk.jmh.infra.Blackhole) Scope(org.openjdk.jmh.annotations.Scope) Random(java.util.Random) ImmutableBitmap(org.apache.druid.collections.bitmap.ImmutableBitmap) Warmup(org.openjdk.jmh.annotations.Warmup) ArrayList(java.util.ArrayList) OutputTimeUnit(org.openjdk.jmh.annotations.OutputTimeUnit) BitmapFactory(org.apache.druid.collections.bitmap.BitmapFactory) IAE(org.apache.druid.java.util.common.IAE) MutableBitmap(org.apache.druid.collections.bitmap.MutableBitmap) Setup(org.openjdk.jmh.annotations.Setup) Mode(org.openjdk.jmh.annotations.Mode) Param(org.openjdk.jmh.annotations.Param) IOException(java.io.IOException) State(org.openjdk.jmh.annotations.State) ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory) Benchmark(org.openjdk.jmh.annotations.Benchmark) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) TimeUnit(java.util.concurrent.TimeUnit) List(java.util.List) NullHandling(org.apache.druid.common.config.NullHandling) Level(org.openjdk.jmh.annotations.Level) Fork(org.openjdk.jmh.annotations.Fork) ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory) MutableBitmap(org.apache.druid.collections.bitmap.MutableBitmap) IAE(org.apache.druid.java.util.common.IAE) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) Setup(org.openjdk.jmh.annotations.Setup)

Example 10 with ConciseBitmapFactory

use of org.apache.druid.collections.bitmap.ConciseBitmapFactory in project druid by druid-io.

the class RangeBitmapBenchmark method setup.

@Setup(Level.Trial)
public void setup() throws IOException {
    switch(type) {
        case "concise":
            bitmapFactory = new ConciseBitmapFactory();
            break;
        case "roaring":
            bitmapFactory = new RoaringBitmapFactory();
            break;
        default:
            throw new IAE("Unknown bitmap type[%s]", type);
    }
    bitmaps = new ArrayList<>(numBitmaps);
    for (int i = 0; i < numBitmaps; ++i) {
        final MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
        int k = 0;
        boolean fill = true;
        while (k < bitmapLength) {
            int runLength = (int) (bitmapLength * density) + RANDOM.nextInt((int) (bitmapLength * density));
            for (int j = k; fill && j < bitmapLength && j < k + runLength; ++j) {
                mutableBitmap.add(j);
            }
            k += runLength;
            fill = !fill;
        }
        for (k = bitmapLength / 2; k < bitmapLength / 2 + minIntersect; ++k) {
            mutableBitmap.add(k);
        }
        bitmaps.add(BitmapBenchmarkUtils.toOffheap(bitmapFactory.makeImmutableBitmap(mutableBitmap)));
    }
    final long totalSizeBytes = bitmaps.stream().mapToLong(bitmap -> bitmap.toBytes().length).sum();
    BitmapBenchmarkUtils.printSizeStats(type, density, bitmaps.size(), totalSizeBytes);
}
Also used : BenchmarkMode(org.openjdk.jmh.annotations.BenchmarkMode) Measurement(org.openjdk.jmh.annotations.Measurement) Blackhole(org.openjdk.jmh.infra.Blackhole) Scope(org.openjdk.jmh.annotations.Scope) Random(java.util.Random) ImmutableBitmap(org.apache.druid.collections.bitmap.ImmutableBitmap) Warmup(org.openjdk.jmh.annotations.Warmup) ArrayList(java.util.ArrayList) OutputTimeUnit(org.openjdk.jmh.annotations.OutputTimeUnit) BitmapFactory(org.apache.druid.collections.bitmap.BitmapFactory) IAE(org.apache.druid.java.util.common.IAE) MutableBitmap(org.apache.druid.collections.bitmap.MutableBitmap) Setup(org.openjdk.jmh.annotations.Setup) Mode(org.openjdk.jmh.annotations.Mode) Param(org.openjdk.jmh.annotations.Param) IOException(java.io.IOException) State(org.openjdk.jmh.annotations.State) ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory) Benchmark(org.openjdk.jmh.annotations.Benchmark) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) TimeUnit(java.util.concurrent.TimeUnit) List(java.util.List) NullHandling(org.apache.druid.common.config.NullHandling) Level(org.openjdk.jmh.annotations.Level) Fork(org.openjdk.jmh.annotations.Fork) ConciseBitmapFactory(org.apache.druid.collections.bitmap.ConciseBitmapFactory) MutableBitmap(org.apache.druid.collections.bitmap.MutableBitmap) IAE(org.apache.druid.java.util.common.IAE) RoaringBitmapFactory(org.apache.druid.collections.bitmap.RoaringBitmapFactory) Setup(org.openjdk.jmh.annotations.Setup)

Aggregations

ConciseBitmapFactory (org.apache.druid.collections.bitmap.ConciseBitmapFactory)18 RoaringBitmapFactory (org.apache.druid.collections.bitmap.RoaringBitmapFactory)17 BitmapFactory (org.apache.druid.collections.bitmap.BitmapFactory)16 ImmutableBitmap (org.apache.druid.collections.bitmap.ImmutableBitmap)13 LinearGutmanSplitStrategy (org.apache.druid.collections.spatial.split.LinearGutmanSplitStrategy)11 Test (org.junit.Test)11 Random (java.util.Random)10 ThreadLocalRandom (java.util.concurrent.ThreadLocalRandom)8 IntIterator (org.roaringbitmap.IntIterator)8 RadiusBound (org.apache.druid.collections.spatial.search.RadiusBound)7 IOException (java.io.IOException)3 MutableBitmap (org.apache.druid.collections.bitmap.MutableBitmap)3 IAE (org.apache.druid.java.util.common.IAE)3 Setup (org.openjdk.jmh.annotations.Setup)3 Stopwatch (com.google.common.base.Stopwatch)2 ArrayList (java.util.ArrayList)2 List (java.util.List)2 TimeUnit (java.util.concurrent.TimeUnit)2 Point (org.apache.druid.collections.spatial.Point)2 RectangularBound (org.apache.druid.collections.spatial.search.RectangularBound)2