Examples with BaseColumn org.apache.druid.segment.column.BaseColumn used on opensource projects

Example 1 with BaseColumn

use of org.apache.druid.segment.column.BaseColumn in project druid by druid-io.

the class QueryableIndexIndexableAdapter method getDimValueLookup.

@Nullable
@Override
public <T extends Comparable<? super T>> CloseableIndexed<T> getDimValueLookup(String dimension) {
    final ColumnHolder columnHolder = input.getColumnHolder(dimension);
    if (columnHolder == null) {
        return null;
    }
    final BaseColumn col = columnHolder.getColumn();
    if (!(col instanceof DictionaryEncodedColumn)) {
        return null;
    }
    @SuppressWarnings("unchecked") DictionaryEncodedColumn<T> dict = (DictionaryEncodedColumn<T>) col;
    return new CloseableIndexed<T>() {

        @Override
        public int size() {
            return dict.getCardinality();
        }

        @Override
        public T get(int index) {
            return dict.lookupName(index);
        }

        @Override
        public int indexOf(T value) {
            return dict.lookupId(value);
        }

        @Override
        public Iterator<T> iterator() {
            return IndexedIterable.create(this).iterator();
        }

        @Override
        public void inspectRuntimeShape(RuntimeShapeInspector inspector) {
            inspector.visit("dict", dict);
        }

        @Override
        public void close() throws IOException {
            dict.close();
        }
    };
}

Example 2 with BaseColumn

use of org.apache.druid.segment.column.BaseColumn in project druid by druid-io.

the class ColumnSelectorBitmapIndexSelector method getDimensionValues.

@Nullable
@Override
public CloseableIndexed<String> getDimensionValues(String dimension) {
    if (isVirtualColumn(dimension)) {
        BitmapIndex bitmapIndex = virtualColumns.getBitmapIndex(dimension, index);
        if (bitmapIndex == null) {
            return null;
        }
        return new CloseableIndexed<String>() {

            @Override
            public int size() {
                return bitmapIndex.getCardinality();
            }

            @Override
            public String get(int index) {
                return bitmapIndex.getValue(index);
            }

            @Override
            public int indexOf(String value) {
                return bitmapIndex.getIndex(value);
            }

            @Override
            public Iterator<String> iterator() {
                return IndexedIterable.create(this).iterator();
            }

            @Override
            public void inspectRuntimeShape(RuntimeShapeInspector inspector) {
                inspector.visit("column", bitmapIndex);
            }

            @Override
            public void close() {
            }
        };
    }
    final ColumnHolder columnHolder = index.getColumnHolder(dimension);
    if (columnHolder == null) {
        return null;
    }
    if (!columnHolder.getCapabilities().toColumnType().is(ValueType.STRING)) {
        // work correctly here until reworking is done to support filtering/indexing other types of columns
        return null;
    }
    BaseColumn col = columnHolder.getColumn();
    if (!(col instanceof DictionaryEncodedColumn)) {
        return null;
    }
    final DictionaryEncodedColumn<String> column = (DictionaryEncodedColumn<String>) col;
    return new CloseableIndexed<String>() {

        @Override
        public int size() {
            return column.getCardinality();
        }

        @Override
        public String get(int index) {
            return column.lookupName(index);
        }

        @Override
        public int indexOf(String value) {
            return column.lookupId(value);
        }

        @Override
        public Iterator<String> iterator() {
            return IndexedIterable.create(this).iterator();
        }

        @Override
        public void inspectRuntimeShape(RuntimeShapeInspector inspector) {
            inspector.visit("column", column);
        }

        @Override
        public void close() throws IOException {
            column.close();
        }
    };
}

Example 3 with BaseColumn

use of org.apache.druid.segment.column.BaseColumn in project druid by druid-io.

the class BroadcastSegmentIndexedTableTest method checkColumnSelectorFactory.

private void checkColumnSelectorFactory(String columnName) {
    try (final Closer closer = Closer.create()) {
        final int numRows = backingSegment.asStorageAdapter().getNumRows();
        final SimpleAscendingOffset offset = new SimpleAscendingOffset(numRows);
        final BaseColumn theColumn = backingSegment.asQueryableIndex().getColumnHolder(columnName).getColumn();
        closer.register(theColumn);
        final BaseObjectColumnValueSelector<?> selector = theColumn.makeColumnValueSelector(offset);
        ColumnSelectorFactory tableFactory = broadcastTable.makeColumnSelectorFactory(offset, false, closer);
        final BaseObjectColumnValueSelector<?> tableSelector = tableFactory.makeColumnValueSelector(columnName);
        // compare with base segment selector to make sure tables selector can read correct values
        for (int row = 0; row < numRows; row++) {
            offset.setCurrentOffset(row);
            Assert.assertEquals(selector.getObject(), tableSelector.getObject());
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

Example 4 with BaseColumn

use of org.apache.druid.segment.column.BaseColumn in project druid by druid-io.

the class QueryableIndexCursorSequenceBuilder method buildVectorized.

public VectorCursor buildVectorized(final int vectorSize) {
    // Sanity check - matches QueryableIndexStorageAdapter.canVectorize
    Preconditions.checkState(!descending, "!descending");
    final Map<String, BaseColumn> columnCache = new HashMap<>();
    final Closer closer = Closer.create();
    NumericColumn timestamps = null;
    final int startOffset;
    final int endOffset;
    if (interval.getStartMillis() > minDataTimestamp) {
        timestamps = (NumericColumn) index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getColumn();
        closer.register(timestamps);
        startOffset = timeSearch(timestamps, interval.getStartMillis(), 0, index.getNumRows());
    } else {
        startOffset = 0;
    }
    if (interval.getEndMillis() <= maxDataTimestamp) {
        if (timestamps == null) {
            timestamps = (NumericColumn) index.getColumnHolder(ColumnHolder.TIME_COLUMN_NAME).getColumn();
            closer.register(timestamps);
        }
        endOffset = timeSearch(timestamps, interval.getEndMillis(), startOffset, index.getNumRows());
    } else {
        endOffset = index.getNumRows();
    }
    final VectorOffset baseOffset = filterBitmap == null ? new NoFilterVectorOffset(vectorSize, startOffset, endOffset) : new BitmapVectorOffset(vectorSize, filterBitmap, startOffset, endOffset);
    // baseColumnSelectorFactory using baseOffset is the column selector for filtering.
    final VectorColumnSelectorFactory baseColumnSelectorFactory = makeVectorColumnSelectorFactoryForOffset(columnCache, baseOffset, closer);
    if (postFilter == null) {
        return new QueryableIndexVectorCursor(baseColumnSelectorFactory, baseOffset, vectorSize, closer);
    } else {
        final VectorOffset filteredOffset = FilteredVectorOffset.create(baseOffset, baseColumnSelectorFactory, postFilter);
        // Now create the cursor and column selector that will be returned to the caller.
        // 
        // There is an inefficiency with how we do things here: this cursor (the one that will be provided to the
        // caller) does share a columnCache with "baseColumnSelectorFactory", but it *doesn't* share vector data. This
        // means that if the caller wants to read from a column that is also used for filtering, the underlying column
        // object will get hit twice for some of the values (anything that matched the filter). This is probably most
        // noticeable if it causes thrashing of decompression buffers due to out-of-order reads. I haven't observed
        // this directly but it seems possible in principle.
        // baseColumnSelectorFactory using baseOffset is the column selector for filtering.
        final VectorColumnSelectorFactory filteredColumnSelectorFactory = makeVectorColumnSelectorFactoryForOffset(columnCache, filteredOffset, closer);
        return new QueryableIndexVectorCursor(filteredColumnSelectorFactory, filteredOffset, vectorSize, closer);
    }
}

Example 5 with BaseColumn

use of org.apache.druid.segment.column.BaseColumn in project druid by druid-io.

the class SegmentAnalyzer method analyzeStringColumn.

private ColumnAnalysis analyzeStringColumn(final ColumnCapabilities capabilities, final ColumnHolder columnHolder) {
    Comparable min = null;
    Comparable max = null;
    long size = 0;
    final int cardinality;
    if (capabilities.hasBitmapIndexes()) {
        final BitmapIndex bitmapIndex = columnHolder.getBitmapIndex();
        cardinality = bitmapIndex.getCardinality();
        if (analyzingSize()) {
            for (int i = 0; i < cardinality; ++i) {
                String value = bitmapIndex.getValue(i);
                if (value != null) {
                    size += StringUtils.estimatedBinaryLengthAsUTF8(value) * ((long) bitmapIndex.getBitmap(bitmapIndex.getIndex(value)).size());
                }
            }
        }
        if (analyzingMinMax() && cardinality > 0) {
            min = NullHandling.nullToEmptyIfNeeded(bitmapIndex.getValue(0));
            max = NullHandling.nullToEmptyIfNeeded(bitmapIndex.getValue(cardinality - 1));
        }
    } else if (capabilities.isDictionaryEncoded().isTrue()) {
        // fallback if no bitmap index
        try (BaseColumn column = columnHolder.getColumn()) {
            DictionaryEncodedColumn<String> theColumn = (DictionaryEncodedColumn<String>) column;
            cardinality = theColumn.getCardinality();
            if (analyzingMinMax() && cardinality > 0) {
                min = NullHandling.nullToEmptyIfNeeded(theColumn.lookupName(0));
                max = NullHandling.nullToEmptyIfNeeded(theColumn.lookupName(cardinality - 1));
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    } else {
        cardinality = 0;
    }
    return new ColumnAnalysis(capabilities.toColumnType(), capabilities.getType().name(), capabilities.hasMultipleValues().isTrue(), // if we don't know for sure, then we should plan to check for nulls
    capabilities.hasNulls().isMaybeTrue(), size, analyzingCardinality() ? cardinality : 0, min, max, null);
}