Examples with ColumnCapabilities org.apache.druid.segment.column.ColumnCapabilities used on opensource projects

Example 36 with ColumnCapabilities

use of org.apache.druid.segment.column.ColumnCapabilities in project druid by druid-io.

the class DimensionHandlerUtilsTest method testGetHandlerFromDoubleCapabilities.

@Test
public void testGetHandlerFromDoubleCapabilities() {
    ColumnCapabilities capabilities = ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.DOUBLE);
    DimensionHandler handler = DimensionHandlerUtils.getHandlerFromCapabilities(DIM_NAME, capabilities, null);
    Assert.assertTrue(handler instanceof DoubleDimensionHandler);
    Assert.assertTrue(handler.getDimensionSchema(capabilities) instanceof DoubleDimensionSchema);
}

Example 37 with ColumnCapabilities

use of org.apache.druid.segment.column.ColumnCapabilities in project druid by druid-io.

the class IndexMergerV9 method makeDimensionHandlers.

private Map<String, DimensionHandler> makeDimensionHandlers(final List<String> mergedDimensions, final List<ColumnCapabilities> dimCapabilities) {
    Map<String, DimensionHandler> handlers = new LinkedHashMap<>();
    for (int i = 0; i < mergedDimensions.size(); i++) {
        ColumnCapabilities capabilities = ColumnCapabilitiesImpl.snapshot(dimCapabilities.get(i), DIMENSION_CAPABILITY_MERGE_LOGIC);
        String dimName = mergedDimensions.get(i);
        DimensionHandler handler = DimensionHandlerUtils.getHandlerFromCapabilities(dimName, capabilities, null);
        handlers.put(dimName, handler);
    }
    return handlers;
}

Example 38 with ColumnCapabilities

use of org.apache.druid.segment.column.ColumnCapabilities in project druid by druid-io.

the class IndexMergerV9 method makeIndexFiles.

private File makeIndexFiles(final List<IndexableAdapter> adapters, @Nullable final AggregatorFactory[] metricAggs, final File outDir, final ProgressIndicator progress, final List<String> mergedDimensions, final List<String> mergedMetrics, final Function<List<TransformableRowIterator>, TimeAndDimsIterator> rowMergerFn, final boolean fillRowNumConversions, final IndexSpec indexSpec, @Nullable final SegmentWriteOutMediumFactory segmentWriteOutMediumFactory) throws IOException {
    progress.start();
    progress.progress();
    List<Metadata> metadataList = Lists.transform(adapters, IndexableAdapter::getMetadata);
    final Metadata segmentMetadata;
    if (metricAggs != null) {
        AggregatorFactory[] combiningMetricAggs = new AggregatorFactory[metricAggs.length];
        for (int i = 0; i < metricAggs.length; i++) {
            combiningMetricAggs[i] = metricAggs[i].getCombiningFactory();
        }
        segmentMetadata = Metadata.merge(metadataList, combiningMetricAggs);
    } else {
        segmentMetadata = Metadata.merge(metadataList, null);
    }
    Closer closer = Closer.create();
    try {
        final FileSmoosher v9Smoosher = new FileSmoosher(outDir);
        FileUtils.mkdirp(outDir);
        SegmentWriteOutMediumFactory omf = segmentWriteOutMediumFactory != null ? segmentWriteOutMediumFactory : defaultSegmentWriteOutMediumFactory;
        log.debug("Using SegmentWriteOutMediumFactory[%s]", omf.getClass().getSimpleName());
        SegmentWriteOutMedium segmentWriteOutMedium = omf.makeSegmentWriteOutMedium(outDir);
        closer.register(segmentWriteOutMedium);
        long startTime = System.currentTimeMillis();
        Files.asByteSink(new File(outDir, "version.bin")).write(Ints.toByteArray(IndexIO.V9_VERSION));
        log.debug("Completed version.bin in %,d millis.", System.currentTimeMillis() - startTime);
        progress.progress();
        startTime = System.currentTimeMillis();
        try (FileOutputStream fos = new FileOutputStream(new File(outDir, "factory.json"))) {
            SegmentizerFactory customSegmentLoader = indexSpec.getSegmentLoader();
            if (customSegmentLoader != null) {
                mapper.writeValue(fos, customSegmentLoader);
            } else {
                mapper.writeValue(fos, new MMappedQueryableSegmentizerFactory(indexIO));
            }
        }
        log.debug("Completed factory.json in %,d millis", System.currentTimeMillis() - startTime);
        progress.progress();
        final Map<String, ValueType> metricsValueTypes = new TreeMap<>(Comparators.naturalNullsFirst());
        final Map<String, String> metricTypeNames = new TreeMap<>(Comparators.naturalNullsFirst());
        final List<ColumnCapabilities> dimCapabilities = Lists.newArrayListWithCapacity(mergedDimensions.size());
        mergeCapabilities(adapters, mergedDimensions, metricsValueTypes, metricTypeNames, dimCapabilities);
        final Map<String, DimensionHandler> handlers = makeDimensionHandlers(mergedDimensions, dimCapabilities);
        final List<DimensionMergerV9> mergers = new ArrayList<>();
        for (int i = 0; i < mergedDimensions.size(); i++) {
            DimensionHandler handler = handlers.get(mergedDimensions.get(i));
            mergers.add(handler.makeMerger(indexSpec, segmentWriteOutMedium, dimCapabilities.get(i), progress, closer));
        }
        /**
         *********** Setup Dim Conversions *************
         */
        progress.progress();
        startTime = System.currentTimeMillis();
        writeDimValuesAndSetupDimConversion(adapters, progress, mergedDimensions, mergers);
        log.debug("Completed dim conversions in %,d millis.", System.currentTimeMillis() - startTime);
        /**
         *********** Walk through data sets, merge them, and write merged columns ************
         */
        progress.progress();
        final TimeAndDimsIterator timeAndDimsIterator = makeMergedTimeAndDimsIterator(adapters, mergedDimensions, mergedMetrics, rowMergerFn, handlers, mergers);
        closer.register(timeAndDimsIterator);
        final GenericColumnSerializer timeWriter = setupTimeWriter(segmentWriteOutMedium, indexSpec);
        final ArrayList<GenericColumnSerializer> metricWriters = setupMetricsWriters(segmentWriteOutMedium, mergedMetrics, metricsValueTypes, metricTypeNames, indexSpec);
        List<IntBuffer> rowNumConversions = mergeIndexesAndWriteColumns(adapters, progress, timeAndDimsIterator, timeWriter, metricWriters, mergers, fillRowNumConversions);
        /**
         ********** Create Inverted Indexes and Finalize Build Columns ************
         */
        final String section = "build inverted index and columns";
        progress.startSection(section);
        makeTimeColumn(v9Smoosher, progress, timeWriter, indexSpec);
        makeMetricsColumns(v9Smoosher, progress, mergedMetrics, metricsValueTypes, metricTypeNames, metricWriters, indexSpec);
        for (int i = 0; i < mergedDimensions.size(); i++) {
            DimensionMergerV9 merger = mergers.get(i);
            merger.writeIndexes(rowNumConversions);
            if (merger.canSkip()) {
                continue;
            }
            ColumnDescriptor columnDesc = merger.makeColumnDescriptor();
            makeColumn(v9Smoosher, mergedDimensions.get(i), columnDesc);
        }
        progress.stopSection(section);
        /**
         *********** Make index.drd & metadata.drd files *************
         */
        progress.progress();
        makeIndexBinary(v9Smoosher, adapters, outDir, mergedDimensions, mergedMetrics, progress, indexSpec, mergers);
        makeMetadataBinary(v9Smoosher, progress, segmentMetadata);
        v9Smoosher.close();
        progress.stop();
        return outDir;
    } catch (Throwable t) {
        throw closer.rethrow(t);
    } finally {
        closer.close();
    }
}

Example 39 with ColumnCapabilities

use of org.apache.druid.segment.column.ColumnCapabilities in project druid by druid-io.

the class RowBasedGrouperHelper method createResultRowBasedColumnSelectorFactory.

/**
 * Creates a {@link ColumnSelectorFactory} that can read rows which originate as results of the provided "query".
 *
 * @param query        a groupBy query
 * @param supplier     supplier of result rows from the query
 * @param finalization whether the column capabilities reported by this factory should reflect finalized types
 */
public static ColumnSelectorFactory createResultRowBasedColumnSelectorFactory(final GroupByQuery query, final Supplier<ResultRow> supplier, final RowSignature.Finalization finalization) {
    final RowSignature signature = query.getResultRowSignature(finalization);
    final RowAdapter<ResultRow> adapter = new RowAdapter<ResultRow>() {

        @Override
        public ToLongFunction<ResultRow> timestampFunction() {
            if (query.getResultRowHasTimestamp()) {
                return row -> row.getLong(0);
            } else {
                final long timestamp = query.getUniversalTimestamp().getMillis();
                return row -> timestamp;
            }
        }

        @Override
        public Function<ResultRow, Object> columnFunction(final String columnName) {
            final int columnIndex = signature.indexOf(columnName);
            if (columnIndex < 0) {
                return row -> null;
            } else {
                return row -> row.get(columnIndex);
            }
        }
    };
    // Decorate "signature" so that it returns hasMultipleValues = false. (groupBy does not return multiple values.)
    final ColumnInspector decoratedSignature = new ColumnInspector() {

        @Nullable
        @Override
        public ColumnCapabilities getColumnCapabilities(String column) {
            final ColumnCapabilities baseCapabilities = signature.getColumnCapabilities(column);
            if (baseCapabilities == null || baseCapabilities.hasMultipleValues().isFalse()) {
                return baseCapabilities;
            } else {
                return ColumnCapabilitiesImpl.copyOf(baseCapabilities).setHasMultipleValues(false);
            }
        }
    };
    return RowBasedColumnSelectorFactory.create(adapter, supplier::get, decoratedSignature, false);
}

Example 40 with ColumnCapabilities

use of org.apache.druid.segment.column.ColumnCapabilities in project druid by druid-io.

the class ApproximateHistogramVectorAggregatorTest method setup.

@Before
public void setup() {
    NullHandling.initializeForTests();
    VectorValueSelector vectorValueSelector_1 = createMock(VectorValueSelector.class);
    expect(vectorValueSelector_1.getFloatVector()).andReturn(FLOATS).anyTimes();
    expect(vectorValueSelector_1.getNullVector()).andReturn(NULL_VECTOR).anyTimes();
    VectorValueSelector vectorValueSelector_2 = createMock(VectorValueSelector.class);
    expect(vectorValueSelector_2.getFloatVector()).andReturn(FLOATS).anyTimes();
    expect(vectorValueSelector_2.getNullVector()).andReturn(null).anyTimes();
    EasyMock.replay(vectorValueSelector_1);
    EasyMock.replay(vectorValueSelector_2);
    ColumnCapabilities columnCapabilities = ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ColumnType.DOUBLE);
    vectorColumnSelectorFactory = createMock(VectorColumnSelectorFactory.class);
    expect(vectorColumnSelectorFactory.getColumnCapabilities("field_1")).andReturn(columnCapabilities).anyTimes();
    expect(vectorColumnSelectorFactory.makeValueSelector("field_1")).andReturn(vectorValueSelector_1).anyTimes();
    expect(vectorColumnSelectorFactory.getColumnCapabilities("field_2")).andReturn(columnCapabilities).anyTimes();
    expect(vectorColumnSelectorFactory.makeValueSelector("field_2")).andReturn(vectorValueSelector_2).anyTimes();
    expect(vectorColumnSelectorFactory.getColumnCapabilities("string_field")).andReturn(new ColumnCapabilitiesImpl().setType(ColumnType.STRING));
    expect(vectorColumnSelectorFactory.getColumnCapabilities("complex_field")).andReturn(new ColumnCapabilitiesImpl().setType(ApproximateHistogramAggregatorFactory.TYPE));
    EasyMock.replay(vectorColumnSelectorFactory);
}