Examples with RTree io.druid.collections.spatial.RTree used on opensource projects

Example 1 with RTree

use of io.druid.collections.spatial.RTree in project druid by druid-io.

the class StringDimensionMergerLegacy method writeIndexes.

@Override
public void writeIndexes(List<IntBuffer> segmentRowNumConversions, Closer closer) throws IOException {
    final SerializerUtils serializerUtils = new SerializerUtils();
    long dimStartTime = System.currentTimeMillis();
    final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory();
    String bmpFilename = String.format("%s.inverted", dimensionName);
    bitmapWriter = new GenericIndexedWriter<>(ioPeon, bmpFilename, bitmapSerdeFactory.getObjectStrategy());
    bitmapWriter.open();
    final MappedByteBuffer dimValsMapped = Files.map(dictionaryFile);
    closer.register(new Closeable() {

        @Override
        public void close() throws IOException {
            ByteBufferUtils.unmap(dimValsMapped);
        }
    });
    if (!dimensionName.equals(serializerUtils.readString(dimValsMapped))) {
        throw new ISE("dimensions[%s] didn't equate!?  This is a major WTF moment.", dimensionName);
    }
    Indexed<String> dimVals = GenericIndexed.read(dimValsMapped, GenericIndexed.STRING_STRATEGY);
    log.info("Starting dimension[%s] with cardinality[%,d]", dimensionName, dimVals.size());
    final BitmapFactory bmpFactory = bitmapSerdeFactory.getBitmapFactory();
    RTree tree = null;
    spatialWriter = null;
    boolean hasSpatial = capabilities.hasSpatialIndexes();
    if (hasSpatial) {
        String spatialFilename = String.format("%s.spatial", dimensionName);
        spatialWriter = new ByteBufferWriter<>(ioPeon, spatialFilename, new IndexedRTree.ImmutableRTreeObjectStrategy(bmpFactory));
        spatialWriter.open();
        tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bmpFactory), bmpFactory);
    }
    IndexSeeker[] dictIdSeeker = toIndexSeekers(adapters, dimConversions, dimensionName);
    //Iterate all dim values's dictionary id in ascending order which in line with dim values's compare result.
    for (int dictId = 0; dictId < dimVals.size(); dictId++) {
        progress.progress();
        mergeBitmaps(segmentRowNumConversions, dimVals, bmpFactory, tree, hasSpatial, dictIdSeeker, dictId, adapters, dimensionName, nullRowsBitmap, bitmapWriter);
    }
    log.info("Completed dimension[%s] in %,d millis.", dimensionName, System.currentTimeMillis() - dimStartTime);
    if (hasSpatial) {
        spatialWriter.write(ImmutableRTree.newImmutableFromMutable(tree));
    }
}

Example 2 with RTree

use of io.druid.collections.spatial.RTree in project druid by druid-io.

the class LinearGutmanSplitStrategyTest method testNumChildrenSizeRoaring.

@Test
public void testNumChildrenSizeRoaring() {
    BitmapFactory bf = new RoaringBitmapFactory();
    RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf);
    Random rand = new Random();
    for (int i = 0; i < 100; i++) {
        tree.insert(new float[] { rand.nextFloat(), rand.nextFloat() }, i);
    }
    Assert.assertTrue(getNumPoints(tree.getRoot()) >= tree.getSize());
}

Example 3 with RTree

use of io.druid.collections.spatial.RTree in project druid by druid-io.

the class StringDimensionMergerV9 method writeIndexes.

@Override
public void writeIndexes(List<IntBuffer> segmentRowNumConversions, Closer closer) throws IOException {
    long dimStartTime = System.currentTimeMillis();
    final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory();
    String bmpFilename = String.format("%s.inverted", dimensionName);
    bitmapWriter = new GenericIndexedWriter<>(ioPeon, bmpFilename, bitmapSerdeFactory.getObjectStrategy());
    bitmapWriter.open();
    // write dim values to one single file because we need to read it
    File dimValueFile = IndexIO.makeDimFile(outDir, dimensionName);
    try (FileOutputStream fos = new FileOutputStream(dimValueFile)) {
        ByteStreams.copy(dictionaryWriter.combineStreams(), fos);
    }
    final MappedByteBuffer dimValsMapped = Files.map(dimValueFile);
    try (Closeable toCloseEncodedValueWriter = encodedValueWriter;
        Closeable toCloseBitmapWriter = bitmapWriter;
        Closeable dimValsMappedUnmapper = new Closeable() {

            @Override
            public void close() {
                ByteBufferUtils.unmap(dimValsMapped);
            }
        }) {
        Indexed<String> dimVals = GenericIndexed.read(dimValsMapped, GenericIndexed.STRING_STRATEGY);
        BitmapFactory bmpFactory = bitmapSerdeFactory.getBitmapFactory();
        RTree tree = null;
        boolean hasSpatial = capabilities.hasSpatialIndexes();
        if (hasSpatial) {
            spatialWriter = new ByteBufferWriter<>(ioPeon, String.format("%s.spatial", dimensionName), new IndexedRTree.ImmutableRTreeObjectStrategy(bmpFactory));
            spatialWriter.open();
            tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bmpFactory), bmpFactory);
        }
        IndexSeeker[] dictIdSeeker = toIndexSeekers(adapters, dimConversions, dimensionName);
        //Iterate all dim values's dictionary id in ascending order which in line with dim values's compare result.
        for (int dictId = 0; dictId < dimVals.size(); dictId++) {
            progress.progress();
            mergeBitmaps(segmentRowNumConversions, dimVals, bmpFactory, tree, hasSpatial, dictIdSeeker, dictId, adapters, dimensionName, nullRowsBitmap, bitmapWriter);
        }
        if (hasSpatial) {
            spatialWriter.write(ImmutableRTree.newImmutableFromMutable(tree));
            spatialWriter.close();
        }
        log.info("Completed dim[%s] inverted with cardinality[%,d] in %,d millis.", dimensionName, dimVals.size(), System.currentTimeMillis() - dimStartTime);
    }
}

Example 4 with RTree

use of io.druid.collections.spatial.RTree in project druid by druid-io.

the class LinearGutmanSplitStrategyTest method testNumChildrenSize.

@Test
public void testNumChildrenSize() {
    BitmapFactory bf = new ConciseBitmapFactory();
    RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf);
    Random rand = new Random();
    for (int i = 0; i < 100; i++) {
        tree.insert(new float[] { rand.nextFloat(), rand.nextFloat() }, i);
    }
    Assert.assertTrue(getNumPoints(tree.getRoot()) >= tree.getSize());
}