Examples with ComplexColumn io.druid.segment.column.ComplexColumn used on opensource projects

Example 1 with ComplexColumn

use of io.druid.segment.column.ComplexColumn in project druid by druid-io.

the class SegmentAnalyzer method analyzeComplexColumn.

private ColumnAnalysis analyzeComplexColumn(@Nullable final ColumnCapabilities capabilities, @Nullable final Column column, final String typeName) {
    try (final ComplexColumn complexColumn = column != null ? column.getComplexColumn() : null) {
        final boolean hasMultipleValues = capabilities != null && capabilities.hasMultipleValues();
        long size = 0;
        if (analyzingSize() && complexColumn != null) {
            final ComplexMetricSerde serde = ComplexMetrics.getSerdeForType(typeName);
            if (serde == null) {
                return ColumnAnalysis.error(String.format("unknown_complex_%s", typeName));
            }
            final Function<Object, Long> inputSizeFn = serde.inputSizeFn();
            if (inputSizeFn == null) {
                return new ColumnAnalysis(typeName, hasMultipleValues, 0, null, null, null, null);
            }
            final int length = column.getLength();
            for (int i = 0; i < length; ++i) {
                size += inputSizeFn.apply(complexColumn.getRowValue(i));
            }
        }
        return new ColumnAnalysis(typeName, hasMultipleValues, size, null, null, null, null);
    }
}

Example 2 with ComplexColumn

use of io.druid.segment.column.ComplexColumn in project druid by druid-io.

the class QueryableIndexIndexableAdapter method getRows.

@Override
public Iterable<Rowboat> getRows() {
    return new Iterable<Rowboat>() {

        @Override
        public Iterator<Rowboat> iterator() {
            return new Iterator<Rowboat>() {

                final GenericColumn timestamps = input.getColumn(Column.TIME_COLUMN_NAME).getGenericColumn();

                final Closeable[] metrics;

                final Closeable[] columns;

                final Closer closer = Closer.create();

                final int numMetrics = getMetricNames().size();

                final DimensionHandler[] handlers = new DimensionHandler[availableDimensions.size()];

                Collection<DimensionHandler> handlerSet = input.getDimensionHandlers().values();

                int currRow = 0;

                boolean done = false;

                {
                    closer.register(timestamps);
                    handlerSet.toArray(handlers);
                    this.columns = FluentIterable.from(handlerSet).transform(new Function<DimensionHandler, Closeable>() {

                        @Override
                        public Closeable apply(DimensionHandler handler) {
                            Column column = input.getColumn(handler.getDimensionName());
                            return handler.getSubColumn(column);
                        }
                    }).toArray(Closeable.class);
                    for (Closeable column : columns) {
                        closer.register(column);
                    }
                    final Indexed<String> availableMetrics = getMetricNames();
                    metrics = new Closeable[availableMetrics.size()];
                    for (int i = 0; i < metrics.length; ++i) {
                        final Column column = input.getColumn(availableMetrics.get(i));
                        final ValueType type = column.getCapabilities().getType();
                        switch(type) {
                            case FLOAT:
                            case LONG:
                                metrics[i] = column.getGenericColumn();
                                break;
                            case COMPLEX:
                                metrics[i] = column.getComplexColumn();
                                break;
                            default:
                                throw new ISE("Cannot handle type[%s]", type);
                        }
                    }
                    for (Closeable metricColumn : metrics) {
                        closer.register(metricColumn);
                    }
                }

                @Override
                public boolean hasNext() {
                    final boolean hasNext = currRow < numRows;
                    if (!hasNext && !done) {
                        CloseQuietly.close(closer);
                        done = true;
                    }
                    return hasNext;
                }

                @Override
                public Rowboat next() {
                    if (!hasNext()) {
                        throw new NoSuchElementException();
                    }
                    final Object[] dims = new Object[columns.length];
                    int dimIndex = 0;
                    for (final Closeable column : columns) {
                        dims[dimIndex] = handlers[dimIndex].getEncodedKeyComponentFromColumn(column, currRow);
                        dimIndex++;
                    }
                    Object[] metricArray = new Object[numMetrics];
                    for (int i = 0; i < metricArray.length; ++i) {
                        if (metrics[i] instanceof IndexedFloatsGenericColumn) {
                            metricArray[i] = ((GenericColumn) metrics[i]).getFloatSingleValueRow(currRow);
                        } else if (metrics[i] instanceof IndexedLongsGenericColumn) {
                            metricArray[i] = ((GenericColumn) metrics[i]).getLongSingleValueRow(currRow);
                        } else if (metrics[i] instanceof ComplexColumn) {
                            metricArray[i] = ((ComplexColumn) metrics[i]).getRowValue(currRow);
                        }
                    }
                    final Rowboat retVal = new Rowboat(timestamps.getLongSingleValueRow(currRow), dims, metricArray, currRow, handlers);
                    ++currRow;
                    return retVal;
                }

                @Override
                public void remove() {
                    throw new UnsupportedOperationException();
                }
            };
        }
    };
}

Example 3 with ComplexColumn

use of io.druid.segment.column.ComplexColumn in project druid by druid-io.

the class LargeColumnSupportedComplexColumnSerializerTest method testSanity.

@Test
public void testSanity() throws IOException {
    HyperUniquesSerdeForTest serde = new HyperUniquesSerdeForTest(Hashing.murmur3_128());
    int[] cases = { 1000, 5000, 10000, 20000 };
    int[] columnSizes = { Integer.MAX_VALUE, Integer.MAX_VALUE / 2, Integer.MAX_VALUE / 4, 5000 * Longs.BYTES, 2500 * Longs.BYTES };
    for (int columnSize : columnSizes) {
        for (int aCase : cases) {
            File tmpFile = FileUtils.getTempDirectory();
            HyperLogLogCollector baseCollector = HyperLogLogCollector.makeLatestCollector();
            try (IOPeon peon = new TmpFileIOPeon();
                FileSmoosher v9Smoosher = new FileSmoosher(tmpFile)) {
                LargeColumnSupportedComplexColumnSerializer serializer = LargeColumnSupportedComplexColumnSerializer.createWithColumnSize(peon, "test", serde.getObjectStrategy(), columnSize);
                serializer.open();
                for (int i = 0; i < aCase; i++) {
                    HyperLogLogCollector collector = HyperLogLogCollector.makeLatestCollector();
                    byte[] hashBytes = fn.hashLong(i).asBytes();
                    collector.add(hashBytes);
                    baseCollector.fold(collector);
                    serializer.serialize(collector);
                }
                serializer.close();
                try (final SmooshedWriter channel = v9Smoosher.addWithSmooshedWriter("test", serializer.getSerializedSize())) {
                    serializer.writeToChannel(channel, v9Smoosher);
                }
            }
            SmooshedFileMapper mapper = Smoosh.map(tmpFile);
            final ColumnBuilder builder = new ColumnBuilder().setType(ValueType.COMPLEX).setHasMultipleValues(false).setFileMapper(mapper);
            serde.deserializeColumn(mapper.mapFile("test"), builder);
            Column column = builder.build();
            ComplexColumn complexColumn = column.getComplexColumn();
            HyperLogLogCollector collector = HyperLogLogCollector.makeLatestCollector();
            for (int i = 0; i < aCase; i++) {
                collector.fold((HyperLogLogCollector) complexColumn.getRowValue(i));
            }
            Assert.assertEquals(baseCollector.estimateCardinality(), collector.estimateCardinality(), 0.0);
        }
    }
}

Example 4 with ComplexColumn

use of io.druid.segment.column.ComplexColumn in project druid by druid-io.

the class QueryableIndexIndexableAdapter method getMetricType.

@Override
public String getMetricType(String metric) {
    final Column column = input.getColumn(metric);
    final ValueType type = column.getCapabilities().getType();
    switch(type) {
        case FLOAT:
            return "float";
        case LONG:
            return "long";
        case COMPLEX:
            {
                try (ComplexColumn complexColumn = column.getComplexColumn()) {
                    return complexColumn.getTypeName();
                }
            }
        default:
            throw new ISE("Unknown type[%s]", type);
    }
}