Examples with ColumnBuilder io.druid.segment.column.ColumnBuilder used on opensource projects

Example 1 with ColumnBuilder

use of io.druid.segment.column.ColumnBuilder in project druid by druid-io.

the class LargeColumnSupportedComplexColumnSerializerTest method testSanity.

@Test
public void testSanity() throws IOException {
    HyperUniquesSerdeForTest serde = new HyperUniquesSerdeForTest(Hashing.murmur3_128());
    int[] cases = { 1000, 5000, 10000, 20000 };
    int[] columnSizes = { Integer.MAX_VALUE, Integer.MAX_VALUE / 2, Integer.MAX_VALUE / 4, 5000 * Longs.BYTES, 2500 * Longs.BYTES };
    for (int columnSize : columnSizes) {
        for (int aCase : cases) {
            File tmpFile = FileUtils.getTempDirectory();
            HyperLogLogCollector baseCollector = HyperLogLogCollector.makeLatestCollector();
            try (IOPeon peon = new TmpFileIOPeon();
                FileSmoosher v9Smoosher = new FileSmoosher(tmpFile)) {
                LargeColumnSupportedComplexColumnSerializer serializer = LargeColumnSupportedComplexColumnSerializer.createWithColumnSize(peon, "test", serde.getObjectStrategy(), columnSize);
                serializer.open();
                for (int i = 0; i < aCase; i++) {
                    HyperLogLogCollector collector = HyperLogLogCollector.makeLatestCollector();
                    byte[] hashBytes = fn.hashLong(i).asBytes();
                    collector.add(hashBytes);
                    baseCollector.fold(collector);
                    serializer.serialize(collector);
                }
                serializer.close();
                try (final SmooshedWriter channel = v9Smoosher.addWithSmooshedWriter("test", serializer.getSerializedSize())) {
                    serializer.writeToChannel(channel, v9Smoosher);
                }
            }
            SmooshedFileMapper mapper = Smoosh.map(tmpFile);
            final ColumnBuilder builder = new ColumnBuilder().setType(ValueType.COMPLEX).setHasMultipleValues(false).setFileMapper(mapper);
            serde.deserializeColumn(mapper.mapFile("test"), builder);
            Column column = builder.build();
            ComplexColumn complexColumn = column.getComplexColumn();
            HyperLogLogCollector collector = HyperLogLogCollector.makeLatestCollector();
            for (int i = 0; i < aCase; i++) {
                collector.fold((HyperLogLogCollector) complexColumn.getRowValue(i));
            }
            Assert.assertEquals(baseCollector.estimateCardinality(), collector.estimateCardinality(), 0.0);
        }
    }
}

Example 2 with ColumnBuilder

use of io.druid.segment.column.ColumnBuilder in project druid by druid-io.

the class DictionaryEncodedColumnPartSerde method getDeserializer.

@Override
public Deserializer getDeserializer() {
    return new Deserializer() {

        @Override
        public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnConfig) {
            final VERSION rVersion = VERSION.fromByte(buffer.get());
            final int rFlags;
            if (rVersion.compareTo(VERSION.COMPRESSED) >= 0) {
                rFlags = buffer.getInt();
            } else {
                rFlags = rVersion.equals(VERSION.UNCOMPRESSED_MULTI_VALUE) ? Feature.MULTI_VALUE.getMask() : NO_FLAGS;
            }
            final boolean hasMultipleValues = Feature.MULTI_VALUE.isSet(rFlags) || Feature.MULTI_VALUE_V3.isSet(rFlags);
            final GenericIndexed<String> rDictionary = GenericIndexed.read(buffer, GenericIndexed.STRING_STRATEGY, builder.getFileMapper());
            builder.setType(ValueType.STRING);
            final WritableSupplier<IndexedInts> rSingleValuedColumn;
            final WritableSupplier<IndexedMultivalue<IndexedInts>> rMultiValuedColumn;
            if (hasMultipleValues) {
                rMultiValuedColumn = readMultiValuedColum(rVersion, buffer, rFlags, builder.getFileMapper());
                rSingleValuedColumn = null;
            } else {
                rSingleValuedColumn = readSingleValuedColumn(rVersion, buffer, builder.getFileMapper());
                rMultiValuedColumn = null;
            }
            builder.setHasMultipleValues(hasMultipleValues).setDictionaryEncodedColumn(new DictionaryEncodedColumnSupplier(rDictionary, rSingleValuedColumn, rMultiValuedColumn, columnConfig.columnCacheSizeBytes()));
            GenericIndexed<ImmutableBitmap> rBitmaps = GenericIndexed.read(buffer, bitmapSerdeFactory.getObjectStrategy(), builder.getFileMapper());
            builder.setBitmapIndex(new BitmapIndexColumnPartSupplier(bitmapSerdeFactory.getBitmapFactory(), rBitmaps, rDictionary));
            ImmutableRTree rSpatialIndex = null;
            if (buffer.hasRemaining()) {
                rSpatialIndex = ByteBufferSerializer.read(buffer, new IndexedRTree.ImmutableRTreeObjectStrategy(bitmapSerdeFactory.getBitmapFactory()));
                builder.setSpatialIndex(new SpatialIndexColumnPartSupplier(rSpatialIndex));
            }
        }

        private WritableSupplier<IndexedInts> readSingleValuedColumn(VERSION version, ByteBuffer buffer, SmooshedFileMapper fileMapper) {
            switch(version) {
                case UNCOMPRESSED_SINGLE_VALUE:
                    return VSizeIndexedInts.readFromByteBuffer(buffer).asWritableSupplier();
                case COMPRESSED:
                    return CompressedVSizeIntsIndexedSupplier.fromByteBuffer(buffer, byteOrder, fileMapper);
            }
            throw new IAE("Unsupported single-value version[%s]", version);
        }

        private WritableSupplier<IndexedMultivalue<IndexedInts>> readMultiValuedColum(VERSION version, ByteBuffer buffer, int flags, SmooshedFileMapper fileMapper) {
            switch(version) {
                case UNCOMPRESSED_MULTI_VALUE:
                    return VSizeIndexed.readFromByteBuffer(buffer).asWritableSupplier();
                case COMPRESSED:
                    if (Feature.MULTI_VALUE.isSet(flags)) {
                        return CompressedVSizeIndexedSupplier.fromByteBuffer(buffer, byteOrder, fileMapper);
                    } else if (Feature.MULTI_VALUE_V3.isSet(flags)) {
                        return CompressedVSizeIndexedV3Supplier.fromByteBuffer(buffer, byteOrder, fileMapper);
                    } else {
                        throw new IAE("Unrecognized multi-value flag[%d]", flags);
                    }
            }
            throw new IAE("Unsupported multi-value version[%s]", version);
        }
    };
}

Example 3 with ColumnBuilder

use of io.druid.segment.column.ColumnBuilder in project druid by druid-io.

the class FloatGenericColumnPartSerde method getDeserializer.

@Override
public Deserializer getDeserializer() {
    return new Deserializer() {

        @Override
        public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnConfig) {
            final CompressedFloatsIndexedSupplier column = CompressedFloatsIndexedSupplier.fromByteBuffer(buffer, byteOrder, builder.getFileMapper());
            builder.setType(ValueType.FLOAT).setHasMultipleValues(false).setGenericColumn(new FloatGenericColumnSupplier(column, byteOrder));
        }
    };
}

Example 4 with ColumnBuilder

use of io.druid.segment.column.ColumnBuilder in project druid by druid-io.

the class LongGenericColumnPartSerde method getDeserializer.

@Override
public Deserializer getDeserializer() {
    return new Deserializer() {

        @Override
        public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnConfig) {
            final CompressedLongsIndexedSupplier column = CompressedLongsIndexedSupplier.fromByteBuffer(buffer, byteOrder, builder.getFileMapper());
            builder.setType(ValueType.LONG).setHasMultipleValues(false).setGenericColumn(new LongGenericColumnSupplier(column));
        }
    };
}