Examples with DimensionIndexer org.apache.druid.segment.DimensionIndexer used on opensource projects

Example 6 with DimensionIndexer

use of org.apache.druid.segment.DimensionIndexer in project druid by druid-io.

the class IncrementalIndexAdapter method getBitmapValues.

@Override
public BitmapValues getBitmapValues(String dimension, int index) {
    DimensionAccessor accessor = accessors.get(dimension);
    if (accessor == null) {
        return BitmapValues.EMPTY;
    }
    ColumnCapabilities capabilities = accessor.dimensionDesc.getCapabilities();
    DimensionIndexer indexer = accessor.dimensionDesc.getIndexer();
    if (!capabilities.hasBitmapIndexes()) {
        return BitmapValues.EMPTY;
    }
    final int id = (Integer) indexer.getUnsortedEncodedValueFromSorted(index);
    if (id < 0 || id >= indexer.getCardinality()) {
        return BitmapValues.EMPTY;
    }
    MutableBitmap bitmapIndex = accessor.invertedIndexes[id];
    if (bitmapIndex == null) {
        return BitmapValues.EMPTY;
    }
    return new MutableBitmapValues(bitmapIndex);
}

Example 7 with DimensionIndexer

use of org.apache.druid.segment.DimensionIndexer in project druid by druid-io.

the class IncrementalIndexColumnSelectorFactory method makeColumnValueSelector.

@Override
public ColumnValueSelector<?> makeColumnValueSelector(String columnName) {
    if (virtualColumns.exists(columnName)) {
        return virtualColumns.makeColumnValueSelector(columnName, this);
    }
    if (columnName.equals(ColumnHolder.TIME_COLUMN_NAME)) {
        return rowHolder;
    }
    final Integer dimIndex = index.getDimensionIndex(columnName);
    if (dimIndex != null) {
        final IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(columnName);
        final DimensionIndexer indexer = dimensionDesc.getIndexer();
        return indexer.makeColumnValueSelector(rowHolder, dimensionDesc);
    }
    return index.makeMetricColumnValueSelector(columnName, rowHolder);
}

Example 8 with DimensionIndexer

use of org.apache.druid.segment.DimensionIndexer in project druid by druid-io.

the class IncrementalIndex method toIncrementalIndexRow.

@VisibleForTesting
IncrementalIndexRowResult toIncrementalIndexRow(InputRow row) {
    row = formatRow(row);
    if (row.getTimestampFromEpoch() < minTimestamp) {
        throw new IAE("Cannot add row[%s] because it is below the minTimestamp[%s]", row, DateTimes.utc(minTimestamp));
    }
    final List<String> rowDimensions = row.getDimensions();
    Object[] dims;
    List<Object> overflow = null;
    long dimsKeySize = 0;
    List<String> parseExceptionMessages = new ArrayList<>();
    synchronized (dimensionDescs) {
        // all known dimensions are assumed missing until we encounter in the rowDimensions
        Set<String> absentDimensions = Sets.newHashSet(dimensionDescs.keySet());
        // first, process dimension values present in the row
        dims = new Object[dimensionDescs.size()];
        for (String dimension : rowDimensions) {
            if (Strings.isNullOrEmpty(dimension)) {
                continue;
            }
            boolean wasNewDim = false;
            DimensionDesc desc = dimensionDescs.get(dimension);
            if (desc != null) {
                absentDimensions.remove(dimension);
            } else {
                wasNewDim = true;
                desc = addNewDimension(dimension, DimensionHandlerUtils.getHandlerFromCapabilities(dimension, // based on the value to use a better handler
                makeDefaultCapabilitiesFromValueType(ColumnType.STRING), null));
            }
            DimensionIndexer indexer = desc.getIndexer();
            Object dimsKey = null;
            try {
                final EncodedKeyComponent<?> encodedKeyComponent = indexer.processRowValsToUnsortedEncodedKeyComponent(row.getRaw(dimension), true);
                dimsKey = encodedKeyComponent.getComponent();
                dimsKeySize += encodedKeyComponent.getEffectiveSizeBytes();
            } catch (ParseException pe) {
                parseExceptionMessages.add(pe.getMessage());
            }
            if (wasNewDim) {
                // unless this is the first row we are processing, all newly discovered columns will be sparse
                if (maxIngestedEventTime != null) {
                    indexer.setSparseIndexed();
                }
                if (overflow == null) {
                    overflow = new ArrayList<>();
                }
                overflow.add(dimsKey);
            } else if (desc.getIndex() > dims.length || dims[desc.getIndex()] != null) {
                /*
           * index > dims.length requires that we saw this dimension and added it to the dimensionOrder map,
           * otherwise index is null. Since dims is initialized based on the size of dimensionOrder on each call to add,
           * it must have been added to dimensionOrder during this InputRow.
           *
           * if we found an index for this dimension it means we've seen it already. If !(index > dims.length) then
           * we saw it on a previous input row (this its safe to index into dims). If we found a value in
           * the dims array for this index, it means we have seen this dimension already on this input row.
           */
                throw new ISE("Dimension[%s] occurred more than once in InputRow", dimension);
            } else {
                dims[desc.getIndex()] = dimsKey;
            }
        }
        // process any dimensions with missing values in the row
        for (String missing : absentDimensions) {
            dimensionDescs.get(missing).getIndexer().setSparseIndexed();
        }
    }
    if (overflow != null) {
        // Merge overflow and non-overflow
        Object[] newDims = new Object[dims.length + overflow.size()];
        System.arraycopy(dims, 0, newDims, 0, dims.length);
        for (int i = 0; i < overflow.size(); ++i) {
            newDims[dims.length + i] = overflow.get(i);
        }
        dims = newDims;
    }
    long truncated = 0;
    if (row.getTimestamp() != null) {
        truncated = gran.bucketStart(row.getTimestampFromEpoch());
    }
    IncrementalIndexRow incrementalIndexRow = IncrementalIndexRow.createTimeAndDimswithDimsKeySize(Math.max(truncated, minTimestamp), dims, dimensionDescsList, dimsKeySize);
    return new IncrementalIndexRowResult(incrementalIndexRow, parseExceptionMessages);
}

Example 9 with DimensionIndexer

use of org.apache.druid.segment.DimensionIndexer in project druid by druid-io.

the class IncrementalIndexStorageAdapter method getDimensionCardinality.

@Override
public int getDimensionCardinality(String dimension) {
    if (dimension.equals(ColumnHolder.TIME_COLUMN_NAME)) {
        return Integer.MAX_VALUE;
    }
    IncrementalIndex.DimensionDesc desc = index.getDimension(dimension);
    if (desc == null) {
        return 0;
    }
    DimensionIndexer indexer = desc.getIndexer();
    int cardinality = indexer.getCardinality();
    return cardinality != DimensionDictionarySelector.CARDINALITY_UNKNOWN ? cardinality : Integer.MAX_VALUE;
}

Example 10 with DimensionIndexer

use of org.apache.druid.segment.DimensionIndexer in project druid by druid-io.

the class IncrementalIndexStorageAdapter method getMinValue.

@Nullable
@Override
public Comparable getMinValue(String column) {
    IncrementalIndex.DimensionDesc desc = index.getDimension(column);
    if (desc == null) {
        return null;
    }
    DimensionIndexer indexer = desc.getIndexer();
    return indexer.getMinValue();
}