Examples with DimensionHandler org.apache.druid.segment.DimensionHandler used on opensource projects

Example 1 with DimensionHandler

use of org.apache.druid.segment.DimensionHandler in project druid by druid-io.

the class CompactionTask method createDimensionsSpec.

private static DimensionsSpec createDimensionsSpec(List<NonnullPair<QueryableIndex, DataSegment>> queryableIndices) {
    final BiMap<String, Integer> uniqueDims = HashBiMap.create();
    final Map<String, DimensionSchema> dimensionSchemaMap = new HashMap<>();
    // Here, we try to retain the order of dimensions as they were specified since the order of dimensions may be
    // optimized for performance.
    // Dimensions are extracted from the recent segments to olders because recent segments are likely to be queried more
    // frequently, and thus the performance should be optimized for recent ones rather than old ones.
    // sort timelineSegments in order of interval, see https://github.com/apache/druid/pull/9905
    queryableIndices.sort((o1, o2) -> Comparators.intervalsByStartThenEnd().compare(o1.rhs.getInterval(), o2.rhs.getInterval()));
    int index = 0;
    for (NonnullPair<QueryableIndex, DataSegment> pair : Lists.reverse(queryableIndices)) {
        final QueryableIndex queryableIndex = pair.lhs;
        final Map<String, DimensionHandler> dimensionHandlerMap = queryableIndex.getDimensionHandlers();
        for (String dimension : queryableIndex.getAvailableDimensions()) {
            final ColumnHolder columnHolder = Preconditions.checkNotNull(queryableIndex.getColumnHolder(dimension), "Cannot find column for dimension[%s]", dimension);
            if (!uniqueDims.containsKey(dimension)) {
                final DimensionHandler dimensionHandler = Preconditions.checkNotNull(dimensionHandlerMap.get(dimension), "Cannot find dimensionHandler for dimension[%s]", dimension);
                uniqueDims.put(dimension, index++);
                dimensionSchemaMap.put(dimension, createDimensionSchema(dimension, columnHolder.getCapabilities(), dimensionHandler.getMultivalueHandling()));
            }
        }
    }
    final BiMap<Integer, String> orderedDims = uniqueDims.inverse();
    final List<DimensionSchema> dimensionSchemas = IntStream.range(0, orderedDims.size()).mapToObj(i -> {
        final String dimName = orderedDims.get(i);
        return Preconditions.checkNotNull(dimensionSchemaMap.get(dimName), "Cannot find dimension[%s] from dimensionSchemaMap", dimName);
    }).collect(Collectors.toList());
    return new DimensionsSpec(dimensionSchemas);
}

Example 2 with DimensionHandler

use of org.apache.druid.segment.DimensionHandler in project druid by druid-io.

the class IncrementalIndex method loadDimensionIterable.

/**
 * Currently called to initialize IncrementalIndex dimension order during index creation
 * Index dimension ordering could be changed to initialize from DimensionsSpec after resolution of
 * https://github.com/apache/druid/issues/2011
 */
public void loadDimensionIterable(Iterable<String> oldDimensionOrder, Map<String, ColumnCapabilities> oldColumnCapabilities) {
    synchronized (dimensionDescs) {
        if (!dimensionDescs.isEmpty()) {
            throw new ISE("Cannot load dimension order when existing order[%s] is not empty.", dimensionDescs.keySet());
        }
        for (String dim : oldDimensionOrder) {
            if (dimensionDescs.get(dim) == null) {
                ColumnCapabilitiesImpl capabilities = ColumnCapabilitiesImpl.snapshot(oldColumnCapabilities.get(dim), IndexMergerV9.DIMENSION_CAPABILITY_MERGE_LOGIC);
                DimensionHandler handler = DimensionHandlerUtils.getHandlerFromCapabilities(dim, capabilities, null);
                addNewDimension(dim, handler);
            }
        }
    }
}

Example 3 with DimensionHandler

use of org.apache.druid.segment.DimensionHandler in project druid by druid-io.

the class IncrementalIndexRowIterator method makeRowPointer.

private static RowPointer makeRowPointer(IncrementalIndex incrementalIndex, IncrementalIndexRowHolder rowHolder, RowNumCounter rowNumCounter) {
    ColumnSelectorFactory columnSelectorFactory = new IncrementalIndexColumnSelectorFactory(new IncrementalIndexStorageAdapter(incrementalIndex), VirtualColumns.EMPTY, false, rowHolder);
    ColumnValueSelector[] dimensionSelectors = incrementalIndex.getDimensions().stream().map(dim -> {
        ColumnValueSelector selectorWithUnsortedValues = columnSelectorFactory.makeColumnValueSelector(dim.getName());
        return dim.getIndexer().convertUnsortedValuesToSorted(selectorWithUnsortedValues);
    }).toArray(ColumnValueSelector[]::new);
    List<DimensionHandler> dimensionHandlers = incrementalIndex.getDimensions().stream().map(IncrementalIndex.DimensionDesc::getHandler).collect(Collectors.toList());
    ColumnValueSelector[] metricSelectors = incrementalIndex.getMetricNames().stream().map(columnSelectorFactory::makeColumnValueSelector).toArray(ColumnValueSelector[]::new);
    return new RowPointer(rowHolder, dimensionSelectors, dimensionHandlers, metricSelectors, incrementalIndex.getMetricNames(), rowNumCounter);
}

Example 4 with DimensionHandler

use of org.apache.druid.segment.DimensionHandler in project druid by druid-io.

the class OnheapIncrementalIndex method iterableWithPostAggregations.

@Override
public Iterable<Row> iterableWithPostAggregations(@Nullable final List<PostAggregator> postAggs, final boolean descending) {
    final AggregatorFactory[] metrics = getMetricAggs();
    {
        return () -> {
            final List<DimensionDesc> dimensions = getDimensions();
            return Iterators.transform(getFacts().iterator(descending), incrementalIndexRow -> {
                final int rowOffset = incrementalIndexRow.getRowIndex();
                Object[] theDims = incrementalIndexRow.getDims();
                Map<String, Object> theVals = Maps.newLinkedHashMap();
                for (int i = 0; i < theDims.length; ++i) {
                    Object dim = theDims[i];
                    DimensionDesc dimensionDesc = dimensions.get(i);
                    if (dimensionDesc == null) {
                        continue;
                    }
                    String dimensionName = dimensionDesc.getName();
                    DimensionHandler handler = dimensionDesc.getHandler();
                    if (dim == null || handler.getLengthOfEncodedKeyComponent(dim) == 0) {
                        theVals.put(dimensionName, null);
                        continue;
                    }
                    final DimensionIndexer indexer = dimensionDesc.getIndexer();
                    Object rowVals = indexer.convertUnsortedEncodedKeyComponentToActualList(dim);
                    theVals.put(dimensionName, rowVals);
                }
                Aggregator[] aggs = getAggsForRow(rowOffset);
                for (int i = 0; i < aggs.length; ++i) {
                    theVals.put(metrics[i].getName(), aggs[i].get());
                }
                if (postAggs != null) {
                    for (PostAggregator postAgg : postAggs) {
                        theVals.put(postAgg.getName(), postAgg.compute(theVals));
                    }
                }
                return new MapBasedRow(incrementalIndexRow.getTimestamp(), theVals);
            });
        };
    }
}