Examples with ColumnCapabilities io.druid.segment.column.ColumnCapabilities used on opensource projects

Example 6 with ColumnCapabilities

use of io.druid.segment.column.ColumnCapabilities in project druid by druid-io.

the class IndexMergerV9 method mergeCapabilities.

private void mergeCapabilities(final List<IndexableAdapter> adapters, final List<String> mergedDimensions, final Map<String, ValueType> metricsValueTypes, final Map<String, String> metricTypeNames, final List<ColumnCapabilitiesImpl> dimCapabilities) {
    final Map<String, ColumnCapabilitiesImpl> capabilitiesMap = Maps.newHashMap();
    for (IndexableAdapter adapter : adapters) {
        for (String dimension : adapter.getDimensionNames()) {
            ColumnCapabilitiesImpl mergedCapabilities = capabilitiesMap.get(dimension);
            if (mergedCapabilities == null) {
                mergedCapabilities = new ColumnCapabilitiesImpl();
                mergedCapabilities.setType(null);
            }
            capabilitiesMap.put(dimension, mergedCapabilities.merge(adapter.getCapabilities(dimension)));
        }
        for (String metric : adapter.getMetricNames()) {
            ColumnCapabilitiesImpl mergedCapabilities = capabilitiesMap.get(metric);
            ColumnCapabilities capabilities = adapter.getCapabilities(metric);
            if (mergedCapabilities == null) {
                mergedCapabilities = new ColumnCapabilitiesImpl();
            }
            capabilitiesMap.put(metric, mergedCapabilities.merge(capabilities));
            metricsValueTypes.put(metric, capabilities.getType());
            metricTypeNames.put(metric, adapter.getMetricType(metric));
        }
    }
    for (String dim : mergedDimensions) {
        dimCapabilities.add(capabilitiesMap.get(dim));
    }
}

Example 7 with ColumnCapabilities

use of io.druid.segment.column.ColumnCapabilities in project druid by druid-io.

the class IncrementalIndexStorageAdapter method makeCursors.

@Override
public Sequence<Cursor> makeCursors(final Filter filter, final Interval interval, final VirtualColumns virtualColumns, final Granularity gran, final boolean descending) {
    if (index.isEmpty()) {
        return Sequences.empty();
    }
    Interval actualIntervalTmp = interval;
    final Interval dataInterval = new Interval(getMinTime().getMillis(), gran.bucketEnd(getMaxTime()).getMillis());
    if (!actualIntervalTmp.overlaps(dataInterval)) {
        return Sequences.empty();
    }
    if (actualIntervalTmp.getStart().isBefore(dataInterval.getStart())) {
        actualIntervalTmp = actualIntervalTmp.withStart(dataInterval.getStart());
    }
    if (actualIntervalTmp.getEnd().isAfter(dataInterval.getEnd())) {
        actualIntervalTmp = actualIntervalTmp.withEnd(dataInterval.getEnd());
    }
    final Interval actualInterval = actualIntervalTmp;
    Iterable<Interval> iterable = gran.getIterable(actualInterval);
    if (descending) {
        iterable = Lists.reverse(ImmutableList.copyOf(iterable));
    }
    return Sequences.map(Sequences.simple(iterable), new Function<Interval, Cursor>() {

        EntryHolder currEntry = new EntryHolder();

        @Override
        public Cursor apply(@Nullable final Interval interval) {
            final long timeStart = Math.max(interval.getStartMillis(), actualInterval.getStartMillis());
            return new Cursor() {

                private final ValueMatcher filterMatcher = makeFilterMatcher(filter, this);

                private Iterator<Map.Entry<IncrementalIndex.TimeAndDims, Integer>> baseIter;

                private Iterable<Map.Entry<IncrementalIndex.TimeAndDims, Integer>> cursorIterable;

                private boolean emptyRange;

                final DateTime time;

                int numAdvanced = -1;

                boolean done;

                {
                    cursorIterable = index.getFacts().timeRangeIterable(descending, timeStart, Math.min(actualInterval.getEndMillis(), gran.increment(interval.getStart()).getMillis()));
                    emptyRange = !cursorIterable.iterator().hasNext();
                    time = gran.toDateTime(interval.getStartMillis());
                    reset();
                }

                @Override
                public DateTime getTime() {
                    return time;
                }

                @Override
                public void advance() {
                    if (!baseIter.hasNext()) {
                        done = true;
                        return;
                    }
                    while (baseIter.hasNext()) {
                        BaseQuery.checkInterrupted();
                        currEntry.set(baseIter.next());
                        if (filterMatcher.matches()) {
                            return;
                        }
                    }
                    if (!filterMatcher.matches()) {
                        done = true;
                    }
                }

                @Override
                public void advanceUninterruptibly() {
                    if (!baseIter.hasNext()) {
                        done = true;
                        return;
                    }
                    while (baseIter.hasNext()) {
                        if (Thread.currentThread().isInterrupted()) {
                            return;
                        }
                        currEntry.set(baseIter.next());
                        if (filterMatcher.matches()) {
                            return;
                        }
                    }
                    if (!filterMatcher.matches()) {
                        done = true;
                    }
                }

                @Override
                public void advanceTo(int offset) {
                    int count = 0;
                    while (count < offset && !isDone()) {
                        advance();
                        count++;
                    }
                }

                @Override
                public boolean isDone() {
                    return done;
                }

                @Override
                public boolean isDoneOrInterrupted() {
                    return isDone() || Thread.currentThread().isInterrupted();
                }

                @Override
                public void reset() {
                    baseIter = cursorIterable.iterator();
                    if (numAdvanced == -1) {
                        numAdvanced = 0;
                    } else {
                        Iterators.advance(baseIter, numAdvanced);
                    }
                    BaseQuery.checkInterrupted();
                    boolean foundMatched = false;
                    while (baseIter.hasNext()) {
                        currEntry.set(baseIter.next());
                        if (filterMatcher.matches()) {
                            foundMatched = true;
                            break;
                        }
                        numAdvanced++;
                    }
                    done = !foundMatched && (emptyRange || !baseIter.hasNext());
                }

                @Override
                public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec) {
                    if (virtualColumns.exists(dimensionSpec.getDimension())) {
                        return virtualColumns.makeDimensionSelector(dimensionSpec, this);
                    }
                    return dimensionSpec.decorate(makeDimensionSelectorUndecorated(dimensionSpec));
                }

                private DimensionSelector makeDimensionSelectorUndecorated(DimensionSpec dimensionSpec) {
                    final String dimension = dimensionSpec.getDimension();
                    final ExtractionFn extractionFn = dimensionSpec.getExtractionFn();
                    if (dimension.equals(Column.TIME_COLUMN_NAME)) {
                        DimensionSelector selector = new SingleScanTimeDimSelector(makeLongColumnSelector(dimension), extractionFn, descending);
                        return selector;
                    }
                    final IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(dimensionSpec.getDimension());
                    if (dimensionDesc == null) {
                        // not a dimension, column may be a metric
                        ColumnCapabilities capabilities = getColumnCapabilities(dimension);
                        if (capabilities == null) {
                            return NullDimensionSelector.instance();
                        }
                        if (capabilities.getType() == ValueType.LONG) {
                            return new LongWrappingDimensionSelector(makeLongColumnSelector(dimension), extractionFn);
                        }
                        if (capabilities.getType() == ValueType.FLOAT) {
                            return new FloatWrappingDimensionSelector(makeFloatColumnSelector(dimension), extractionFn);
                        }
                        // if we can't wrap the base column, just return a column of all nulls
                        return NullDimensionSelector.instance();
                    } else {
                        final DimensionIndexer indexer = dimensionDesc.getIndexer();
                        return indexer.makeDimensionSelector(dimensionSpec, currEntry, dimensionDesc);
                    }
                }

                @Override
                public FloatColumnSelector makeFloatColumnSelector(String columnName) {
                    if (virtualColumns.exists(columnName)) {
                        return virtualColumns.makeFloatColumnSelector(columnName, this);
                    }
                    final Integer dimIndex = index.getDimensionIndex(columnName);
                    if (dimIndex != null) {
                        final IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(columnName);
                        final DimensionIndexer indexer = dimensionDesc.getIndexer();
                        return indexer.makeFloatColumnSelector(currEntry, dimensionDesc);
                    }
                    final Integer metricIndexInt = index.getMetricIndex(columnName);
                    if (metricIndexInt == null) {
                        return ZeroFloatColumnSelector.instance();
                    }
                    final int metricIndex = metricIndexInt;
                    return new FloatColumnSelector() {

                        @Override
                        public float get() {
                            return index.getMetricFloatValue(currEntry.getValue(), metricIndex);
                        }

                        @Override
                        public void inspectRuntimeShape(RuntimeShapeInspector inspector) {
                            inspector.visit("index", index);
                        }
                    };
                }

                @Override
                public LongColumnSelector makeLongColumnSelector(String columnName) {
                    if (virtualColumns.exists(columnName)) {
                        return virtualColumns.makeLongColumnSelector(columnName, this);
                    }
                    if (columnName.equals(Column.TIME_COLUMN_NAME)) {
                        class TimeLongColumnSelector implements LongColumnSelector {

                            @Override
                            public long get() {
                                return currEntry.getKey().getTimestamp();
                            }

                            @Override
                            public void inspectRuntimeShape(RuntimeShapeInspector inspector) {
                            }
                        }
                        return new TimeLongColumnSelector();
                    }
                    final Integer dimIndex = index.getDimensionIndex(columnName);
                    if (dimIndex != null) {
                        final IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(columnName);
                        final DimensionIndexer indexer = dimensionDesc.getIndexer();
                        return indexer.makeLongColumnSelector(currEntry, dimensionDesc);
                    }
                    final Integer metricIndexInt = index.getMetricIndex(columnName);
                    if (metricIndexInt == null) {
                        return ZeroLongColumnSelector.instance();
                    }
                    final int metricIndex = metricIndexInt;
                    return new LongColumnSelector() {

                        @Override
                        public long get() {
                            return index.getMetricLongValue(currEntry.getValue(), metricIndex);
                        }

                        @Override
                        public void inspectRuntimeShape(RuntimeShapeInspector inspector) {
                            inspector.visit("index", index);
                        }
                    };
                }

                @Override
                public ObjectColumnSelector makeObjectColumnSelector(String column) {
                    if (virtualColumns.exists(column)) {
                        return virtualColumns.makeObjectColumnSelector(column, this);
                    }
                    if (column.equals(Column.TIME_COLUMN_NAME)) {
                        return new ObjectColumnSelector<Long>() {

                            @Override
                            public Class classOfObject() {
                                return Long.TYPE;
                            }

                            @Override
                            public Long get() {
                                return currEntry.getKey().getTimestamp();
                            }
                        };
                    }
                    final Integer metricIndexInt = index.getMetricIndex(column);
                    if (metricIndexInt != null) {
                        final int metricIndex = metricIndexInt;
                        final Class classOfObject = index.getMetricClass(column);
                        return new ObjectColumnSelector() {

                            @Override
                            public Class classOfObject() {
                                return classOfObject;
                            }

                            @Override
                            public Object get() {
                                return index.getMetricObjectValue(currEntry.getValue(), metricIndex);
                            }
                        };
                    }
                    IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(column);
                    if (dimensionDesc == null) {
                        return null;
                    } else {
                        final int dimensionIndex = dimensionDesc.getIndex();
                        final DimensionIndexer indexer = dimensionDesc.getIndexer();
                        return new ObjectColumnSelector<Object>() {

                            @Override
                            public Class classOfObject() {
                                return Object.class;
                            }

                            @Override
                            public Object get() {
                                IncrementalIndex.TimeAndDims key = currEntry.getKey();
                                if (key == null) {
                                    return null;
                                }
                                Object[] dims = key.getDims();
                                if (dimensionIndex >= dims.length) {
                                    return null;
                                }
                                return indexer.convertUnsortedEncodedKeyComponentToActualArrayOrList(dims[dimensionIndex], DimensionIndexer.ARRAY);
                            }
                        };
                    }
                }

                @Nullable
                @Override
                public ColumnCapabilities getColumnCapabilities(String columnName) {
                    if (virtualColumns.exists(columnName)) {
                        return virtualColumns.getColumnCapabilities(columnName);
                    }
                    return index.getCapabilities(columnName);
                }
            };
        }
    });
}

Example 8 with ColumnCapabilities

use of io.druid.segment.column.ColumnCapabilities in project druid by druid-io.

the class Filters method makeValueMatcher.

/**
   * Create a ValueMatcher that applies a predicate to row values.
   * <p>
   * The caller provides a predicate factory that can create a predicate for each value type supported by Druid.
   * See {@link DruidPredicateFactory} for more information.
   * <p>
   * When creating the ValueMatcher, the ValueMatcherFactory implementation should decide what type of predicate
   * to create from the predicate factory based on the ValueType of the specified dimension.
   *
   * @param columnSelectorFactory Selector for columns.
   * @param columnName            The column to filter.
   * @param predicateFactory      Predicate factory
   *
   * @return An object that applies a predicate to row values
   */
public static ValueMatcher makeValueMatcher(final ColumnSelectorFactory columnSelectorFactory, final String columnName, final DruidPredicateFactory predicateFactory) {
    final ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(columnName);
    // This should be folded into the ValueMatcherColumnSelectorStrategy once that can handle LONG typed columns.
    if (capabilities != null && capabilities.getType() == ValueType.LONG) {
        return getLongPredicateMatcher(columnSelectorFactory.makeLongColumnSelector(columnName), predicateFactory.makeLongPredicate());
    }
    final ColumnSelectorPlus<ValueMatcherColumnSelectorStrategy> selector = DimensionHandlerUtils.createColumnSelectorPlus(ValueMatcherColumnSelectorStrategyFactory.instance(), DefaultDimensionSpec.of(columnName), columnSelectorFactory);
    return selector.getColumnSelectorStrategy().makeValueMatcher(selector.getSelector(), predicateFactory);
}

Example 9 with ColumnCapabilities

use of io.druid.segment.column.ColumnCapabilities in project druid by druid-io.

the class ExpressionObjectSelector method createBindings.

private static Expr.ObjectBinding createBindings(ColumnSelectorFactory columnSelectorFactory, Expr expression) {
    final Map<String, Supplier<Number>> suppliers = Maps.newHashMap();
    for (String columnName : Parser.findRequiredBindings(expression)) {
        final ColumnCapabilities columnCapabilities = columnSelectorFactory.getColumnCapabilities(columnName);
        final ValueType nativeType = columnCapabilities != null ? columnCapabilities.getType() : null;
        final Supplier<Number> supplier;
        if (nativeType == ValueType.FLOAT) {
            supplier = supplierFromFloatSelector(columnSelectorFactory.makeFloatColumnSelector(columnName));
        } else if (nativeType == ValueType.LONG) {
            supplier = supplierFromLongSelector(columnSelectorFactory.makeLongColumnSelector(columnName));
        } else if (nativeType == null) {
            // Unknown ValueType. Try making an Object selector and see if that gives us anything useful.
            supplier = supplierFromObjectSelector(columnSelectorFactory.makeObjectColumnSelector(columnName));
        } else {
            // Unhandleable ValueType (possibly STRING or COMPLEX).
            supplier = null;
        }
        if (supplier != null) {
            suppliers.put(columnName, supplier);
        }
    }
    return Parser.withSuppliers(suppliers);
}

Example 10 with ColumnCapabilities

use of io.druid.segment.column.ColumnCapabilities in project druid by druid-io.

the class SegmentAnalyzer method analyze.

public Map<String, ColumnAnalysis> analyze(Segment segment) {
    Preconditions.checkNotNull(segment, "segment");
    // index is null for incremental-index-based segments, but storageAdapter is always available
    final QueryableIndex index = segment.asQueryableIndex();
    final StorageAdapter storageAdapter = segment.asStorageAdapter();
    // get length and column names from storageAdapter
    final int length = storageAdapter.getNumRows();
    final Set<String> columnNames = Sets.newHashSet();
    Iterables.addAll(columnNames, storageAdapter.getAvailableDimensions());
    Iterables.addAll(columnNames, storageAdapter.getAvailableMetrics());
    Map<String, ColumnAnalysis> columns = Maps.newTreeMap();
    for (String columnName : columnNames) {
        final Column column = index == null ? null : index.getColumn(columnName);
        final ColumnCapabilities capabilities = column != null ? column.getCapabilities() : storageAdapter.getColumnCapabilities(columnName);
        final ColumnAnalysis analysis;
        final ValueType type = capabilities.getType();
        switch(type) {
            case LONG:
                analysis = analyzeNumericColumn(capabilities, length, Longs.BYTES);
                break;
            case FLOAT:
                analysis = analyzeNumericColumn(capabilities, length, NUM_BYTES_IN_TEXT_FLOAT);
                break;
            case STRING:
                if (index != null) {
                    analysis = analyzeStringColumn(capabilities, column);
                } else {
                    analysis = analyzeStringColumn(capabilities, storageAdapter, columnName);
                }
                break;
            case COMPLEX:
                analysis = analyzeComplexColumn(capabilities, column, storageAdapter.getColumnTypeName(columnName));
                break;
            default:
                log.warn("Unknown column type[%s].", type);
                analysis = ColumnAnalysis.error(String.format("unknown_type_%s", type));
        }
        columns.put(columnName, analysis);
    }
    // Add time column too
    ColumnCapabilities timeCapabilities = storageAdapter.getColumnCapabilities(Column.TIME_COLUMN_NAME);
    if (timeCapabilities == null) {
        timeCapabilities = new ColumnCapabilitiesImpl().setType(ValueType.LONG).setHasMultipleValues(false);
    }
    columns.put(Column.TIME_COLUMN_NAME, analyzeNumericColumn(timeCapabilities, length, NUM_BYTES_IN_TIMESTAMP));
    return columns;
}