Examples with DictionaryEncodedColumn org.apache.druid.segment.column.DictionaryEncodedColumn used on opensource projects

Example 1 with DictionaryEncodedColumn

use of org.apache.druid.segment.column.DictionaryEncodedColumn in project druid by druid-io.

the class QueryableIndexColumnSelectorFactory method makeDimensionSelectorUndecorated.

private DimensionSelector makeDimensionSelectorUndecorated(DimensionSpec dimensionSpec) {
    final String dimension = dimensionSpec.getDimension();
    final ExtractionFn extractionFn = dimensionSpec.getExtractionFn();
    final ColumnHolder columnHolder = index.getColumnHolder(dimension);
    if (columnHolder == null) {
        return DimensionSelector.constant(null, extractionFn);
    }
    if (dimension.equals(ColumnHolder.TIME_COLUMN_NAME)) {
        return new SingleScanTimeDimensionSelector(makeColumnValueSelector(dimension), extractionFn, descending);
    }
    ColumnCapabilities capabilities = columnHolder.getCapabilities();
    if (columnHolder.getCapabilities().isNumeric()) {
        return ValueTypes.makeNumericWrappingDimensionSelector(capabilities.getType(), makeColumnValueSelector(dimension), extractionFn);
    }
    final DictionaryEncodedColumn column = getCachedColumn(dimension, DictionaryEncodedColumn.class);
    if (column != null) {
        return column.makeDimensionSelector(offset, extractionFn);
    } else {
        return DimensionSelector.constant(null, extractionFn);
    }
}

Example 2 with DictionaryEncodedColumn

use of org.apache.druid.segment.column.DictionaryEncodedColumn in project druid by druid-io.

the class QueryableIndexIndexableAdapter method getDimValueLookup.

@Nullable
@Override
public <T extends Comparable<? super T>> CloseableIndexed<T> getDimValueLookup(String dimension) {
    final ColumnHolder columnHolder = input.getColumnHolder(dimension);
    if (columnHolder == null) {
        return null;
    }
    final BaseColumn col = columnHolder.getColumn();
    if (!(col instanceof DictionaryEncodedColumn)) {
        return null;
    }
    @SuppressWarnings("unchecked") DictionaryEncodedColumn<T> dict = (DictionaryEncodedColumn<T>) col;
    return new CloseableIndexed<T>() {

        @Override
        public int size() {
            return dict.getCardinality();
        }

        @Override
        public T get(int index) {
            return dict.lookupName(index);
        }

        @Override
        public int indexOf(T value) {
            return dict.lookupId(value);
        }

        @Override
        public Iterator<T> iterator() {
            return IndexedIterable.create(this).iterator();
        }

        @Override
        public void inspectRuntimeShape(RuntimeShapeInspector inspector) {
            inspector.visit("dict", dict);
        }

        @Override
        public void close() throws IOException {
            dict.close();
        }
    };
}

Example 3 with DictionaryEncodedColumn

use of org.apache.druid.segment.column.DictionaryEncodedColumn in project druid by druid-io.

the class ColumnSelectorBitmapIndexSelector method getDimensionValues.

@Nullable
@Override
public CloseableIndexed<String> getDimensionValues(String dimension) {
    if (isVirtualColumn(dimension)) {
        BitmapIndex bitmapIndex = virtualColumns.getBitmapIndex(dimension, index);
        if (bitmapIndex == null) {
            return null;
        }
        return new CloseableIndexed<String>() {

            @Override
            public int size() {
                return bitmapIndex.getCardinality();
            }

            @Override
            public String get(int index) {
                return bitmapIndex.getValue(index);
            }

            @Override
            public int indexOf(String value) {
                return bitmapIndex.getIndex(value);
            }

            @Override
            public Iterator<String> iterator() {
                return IndexedIterable.create(this).iterator();
            }

            @Override
            public void inspectRuntimeShape(RuntimeShapeInspector inspector) {
                inspector.visit("column", bitmapIndex);
            }

            @Override
            public void close() {
            }
        };
    }
    final ColumnHolder columnHolder = index.getColumnHolder(dimension);
    if (columnHolder == null) {
        return null;
    }
    if (!columnHolder.getCapabilities().toColumnType().is(ValueType.STRING)) {
        // work correctly here until reworking is done to support filtering/indexing other types of columns
        return null;
    }
    BaseColumn col = columnHolder.getColumn();
    if (!(col instanceof DictionaryEncodedColumn)) {
        return null;
    }
    final DictionaryEncodedColumn<String> column = (DictionaryEncodedColumn<String>) col;
    return new CloseableIndexed<String>() {

        @Override
        public int size() {
            return column.getCardinality();
        }

        @Override
        public String get(int index) {
            return column.lookupName(index);
        }

        @Override
        public int indexOf(String value) {
            return column.lookupId(value);
        }

        @Override
        public Iterator<String> iterator() {
            return IndexedIterable.create(this).iterator();
        }

        @Override
        public void inspectRuntimeShape(RuntimeShapeInspector inspector) {
            inspector.visit("column", column);
        }

        @Override
        public void close() throws IOException {
            column.close();
        }
    };
}

Example 4 with DictionaryEncodedColumn

use of org.apache.druid.segment.column.DictionaryEncodedColumn in project druid by druid-io.

the class IndexMergerTestBase method assertDimCompression.

private void assertDimCompression(QueryableIndex index, CompressionStrategy expectedStrategy) throws Exception {
    // Java voodoo
    if (expectedStrategy == null || expectedStrategy == CompressionStrategy.UNCOMPRESSED) {
        return;
    }
    DictionaryEncodedColumn encodedColumn = (DictionaryEncodedColumn) index.getColumnHolder("dim2").getColumn();
    Object obj;
    if (encodedColumn.hasMultipleValues()) {
        Field field = StringDictionaryEncodedColumn.class.getDeclaredField("multiValueColumn");
        field.setAccessible(true);
        obj = field.get(encodedColumn);
    } else {
        Field field = StringDictionaryEncodedColumn.class.getDeclaredField("column");
        field.setAccessible(true);
        obj = field.get(encodedColumn);
    }
    // CompressedVSizeColumnarIntsSupplier$CompressedByteSizeColumnarInts
    // CompressedVSizeColumnarMultiIntsSupplier$CompressedVSizeColumnarMultiInts
    Field compressedSupplierField = obj.getClass().getDeclaredField("this$0");
    compressedSupplierField.setAccessible(true);
    Object supplier = compressedSupplierField.get(obj);
    Field compressionField = supplier.getClass().getDeclaredField("compression");
    compressionField.setAccessible(true);
    Object strategy = compressionField.get(supplier);
    Assert.assertEquals(expectedStrategy, strategy);
}

Example 5 with DictionaryEncodedColumn

use of org.apache.druid.segment.column.DictionaryEncodedColumn in project druid by druid-io.

the class SegmentAnalyzer method analyzeStringColumn.

private ColumnAnalysis analyzeStringColumn(final ColumnCapabilities capabilities, final ColumnHolder columnHolder) {
    Comparable min = null;
    Comparable max = null;
    long size = 0;
    final int cardinality;
    if (capabilities.hasBitmapIndexes()) {
        final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex();
        cardinality = bitmapIndex.getCardinality();
        if (analyzingSize()) {
            for (int i = 0; i < cardinality; ++i) {
                String value = bitmapIndex.getValue(i);
                if (value != null) {
                    size += StringUtils.estimatedBinaryLengthAsUTF8(value) * ((long) bitmapIndex.getBitmap(bitmapIndex.getIndex(value)).size());
                }
            }
        }
        if (analyzingMinMax() && cardinality > 0) {
            min = NullHandling.nullToEmptyIfNeeded(bitmapIndex.getValue(0));
            max = NullHandling.nullToEmptyIfNeeded(bitmapIndex.getValue(cardinality - 1));
        }
    } else if (capabilities.isDictionaryEncoded().isTrue()) {
        // fallback if no bitmap index
        try (BaseColumn column = columnHolder.getColumn()) {
            DictionaryEncodedColumn<String> theColumn = (DictionaryEncodedColumn<String>) column;
            cardinality = theColumn.getCardinality();
            if (analyzingMinMax() && cardinality > 0) {
                min = NullHandling.nullToEmptyIfNeeded(theColumn.lookupName(0));
                max = NullHandling.nullToEmptyIfNeeded(theColumn.lookupName(cardinality - 1));
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    } else {
        cardinality = 0;
    }
    return new ColumnAnalysis(capabilities.toColumnType(), capabilities.getType().name(), capabilities.hasMultipleValues().isTrue(), // if we don't know for sure, then we should plan to check for nulls
    capabilities.hasNulls().isMaybeTrue(), size, analyzingCardinality() ? cardinality : 0, min, max, null);
}