Examples with ComplexMetricSerde org.apache.druid.segment.serde.ComplexMetricSerde used on opensource projects

Example 1 with ComplexMetricSerde

use of org.apache.druid.segment.serde.ComplexMetricSerde in project druid by druid-io.

the class MetricHolder method fromByteBuffer.

public static MetricHolder fromByteBuffer(ByteBuffer buf) {
    final byte ver = buf.get();
    if (VERSION[0] != ver) {
        throw new ISE("Unknown version[%s] of MetricHolder", ver);
    }
    final String metricName = SERIALIZER_UTILS.readString(buf);
    final String typeName = SERIALIZER_UTILS.readString(buf);
    MetricHolder holder = new MetricHolder(metricName, typeName);
    switch(holder.type) {
        case FLOAT:
            holder.floatType = CompressedColumnarFloatsSupplier.fromByteBuffer(buf, ByteOrder.nativeOrder());
            break;
        case COMPLEX:
            final ComplexMetricSerde serdeForType = ComplexMetrics.getSerdeForType(holder.getTypeName());
            if (serdeForType == null) {
                throw new ISE("Unknown type[%s], cannot load.", holder.getTypeName());
            }
            holder.complexType = read(buf, serdeForType);
            break;
        case LONG:
        case DOUBLE:
            throw new ISE("Unsupported type[%s]", holder.type);
    }
    return holder;
}

Example 2 with ComplexMetricSerde

use of org.apache.druid.segment.serde.ComplexMetricSerde in project druid by druid-io.

the class IncrementalIndex method makeColumnSelectorFactory.

/**
 * Column selector used at ingestion time for inputs to aggregators.
 *
 * @param agg                       the aggregator
 * @param in                        ingestion-time input row supplier
 * @param deserializeComplexMetrics whether complex objects should be deserialized by a {@link ComplexMetricExtractor}
 *
 * @return column selector factory
 */
public static ColumnSelectorFactory makeColumnSelectorFactory(final VirtualColumns virtualColumns, final AggregatorFactory agg, final Supplier<InputRow> in, final boolean deserializeComplexMetrics) {
    // we use RowSignature.empty() because ColumnInspector here should be the InputRow schema, not the
    // IncrementalIndex schema, because we are reading values from the InputRow
    final RowBasedColumnSelectorFactory<InputRow> baseSelectorFactory = RowBasedColumnSelectorFactory.create(RowAdapters.standardRow(), in::get, RowSignature.empty(), true);
    class IncrementalIndexInputRowColumnSelectorFactory implements ColumnSelectorFactory {

        @Override
        public ColumnValueSelector<?> makeColumnValueSelector(final String column) {
            final boolean isComplexMetric = agg.getIntermediateType().is(ValueType.COMPLEX);
            final ColumnValueSelector selector = baseSelectorFactory.makeColumnValueSelector(column);
            if (!isComplexMetric || !deserializeComplexMetrics) {
                return selector;
            } else {
                // Wrap selector in a special one that uses ComplexMetricSerde to modify incoming objects.
                // For complex aggregators that read from multiple columns, we wrap all of them. This is not ideal but it
                // has worked so far.
                final String complexTypeName = agg.getIntermediateType().getComplexTypeName();
                final ComplexMetricSerde serde = ComplexMetrics.getSerdeForType(complexTypeName);
                if (serde == null) {
                    throw new ISE("Don't know how to handle type[%s]", complexTypeName);
                }
                final ComplexMetricExtractor extractor = serde.getExtractor();
                return new ColumnValueSelector() {

                    @Override
                    public boolean isNull() {
                        return selector.isNull();
                    }

                    @Override
                    public long getLong() {
                        return selector.getLong();
                    }

                    @Override
                    public float getFloat() {
                        return selector.getFloat();
                    }

                    @Override
                    public double getDouble() {
                        return selector.getDouble();
                    }

                    @Override
                    public Class classOfObject() {
                        return extractor.extractedClass();
                    }

                    @Nullable
                    @Override
                    public Object getObject() {
                        // Here is where the magic happens: read from "in" directly, don't go through the normal "selector".
                        return extractor.extractValue(in.get(), column, agg);
                    }

                    @Override
                    public void inspectRuntimeShape(RuntimeShapeInspector inspector) {
                        inspector.visit("in", in);
                        inspector.visit("selector", selector);
                        inspector.visit("extractor", extractor);
                    }
                };
            }
        }

        @Override
        public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec) {
            return baseSelectorFactory.makeDimensionSelector(dimensionSpec);
        }

        @Nullable
        @Override
        public ColumnCapabilities getColumnCapabilities(String columnName) {
            return baseSelectorFactory.getColumnCapabilities(columnName);
        }
    }
    return virtualColumns.wrap(new IncrementalIndexInputRowColumnSelectorFactory());
}

Example 3 with ComplexMetricSerde

use of org.apache.druid.segment.serde.ComplexMetricSerde in project druid by druid-io.

the class InputRowSerde method fromBytes.

public static InputRow fromBytes(final Map<String, IndexSerdeTypeHelper> typeHelperMap, byte[] data, AggregatorFactory[] aggs) {
    try {
        ByteArrayDataInput in = ByteStreams.newDataInput(data);
        // Read timestamp
        long timestamp = in.readLong();
        Map<String, Object> event = new HashMap<>();
        // Read dimensions
        List<String> dimensions = new ArrayList<>();
        int dimNum = WritableUtils.readVInt(in);
        for (int i = 0; i < dimNum; i++) {
            String dimension = readString(in);
            dimensions.add(dimension);
            IndexSerdeTypeHelper typeHelper = typeHelperMap.get(dimension);
            if (typeHelper == null) {
                typeHelper = STRING_HELPER;
            }
            Object dimValues = typeHelper.deserialize(in);
            if (dimValues == null) {
                continue;
            }
            if (typeHelper.getType() == ValueType.STRING) {
                List<String> dimensionValues = (List<String>) dimValues;
                if (dimensionValues.size() == 1) {
                    event.put(dimension, dimensionValues.get(0));
                } else {
                    event.put(dimension, dimensionValues);
                }
            } else {
                event.put(dimension, dimValues);
            }
        }
        // Read metrics
        int metricSize = WritableUtils.readVInt(in);
        for (int i = 0; i < metricSize; i++) {
            final String metric = readString(in);
            final AggregatorFactory agg = getAggregator(metric, aggs, i);
            final ColumnType type = agg.getIntermediateType();
            final byte metricNullability = in.readByte();
            if (metricNullability == NullHandling.IS_NULL_BYTE) {
                // metric value is null.
                continue;
            }
            if (type.is(ValueType.FLOAT)) {
                event.put(metric, in.readFloat());
            } else if (type.is(ValueType.LONG)) {
                event.put(metric, WritableUtils.readVLong(in));
            } else if (type.is(ValueType.DOUBLE)) {
                event.put(metric, in.readDouble());
            } else {
                ComplexMetricSerde serde = getComplexMetricSerde(agg.getIntermediateType().getComplexTypeName());
                byte[] value = readBytes(in);
                event.put(metric, serde.fromBytes(value, 0, value.length));
            }
        }
        return new MapBasedInputRow(timestamp, dimensions, event);
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
}

Example 4 with ComplexMetricSerde

use of org.apache.druid.segment.serde.ComplexMetricSerde in project druid by druid-io.

the class SegmentAnalyzer method analyzeComplexColumn.

private ColumnAnalysis analyzeComplexColumn(@Nullable final ColumnCapabilities capabilities, @Nullable final ColumnHolder columnHolder) {
    final TypeSignature<ValueType> typeSignature = capabilities == null ? ColumnType.UNKNOWN_COMPLEX : capabilities;
    final String typeName = typeSignature.getComplexTypeName();
    try (final ComplexColumn complexColumn = columnHolder != null ? (ComplexColumn) columnHolder.getColumn() : null) {
        final boolean hasMultipleValues = capabilities != null && capabilities.hasMultipleValues().isTrue();
        final boolean hasNulls = capabilities != null && capabilities.hasNulls().isMaybeTrue();
        long size = 0;
        if (analyzingSize() && complexColumn != null) {
            final ComplexMetricSerde serde = typeName == null ? null : ComplexMetrics.getSerdeForType(typeName);
            if (serde == null) {
                return ColumnAnalysis.error(StringUtils.format("unknown_complex_%s", typeName));
            }
            final Function<Object, Long> inputSizeFn = serde.inputSizeFn();
            if (inputSizeFn == null) {
                return new ColumnAnalysis(ColumnTypeFactory.ofType(typeSignature), typeName, hasMultipleValues, hasNulls, 0, null, null, null, null);
            }
            final int length = complexColumn.getLength();
            for (int i = 0; i < length; ++i) {
                size += inputSizeFn.apply(complexColumn.getRowValue(i));
            }
        }
        return new ColumnAnalysis(ColumnTypeFactory.ofType(typeSignature), typeName, hasMultipleValues, hasNulls, size, null, null, null, null);
    }
}

Example 5 with ComplexMetricSerde

use of org.apache.druid.segment.serde.ComplexMetricSerde in project druid by druid-io.

the class InputRowSerde method toBytes.

public static SerializeResult toBytes(final Map<String, IndexSerdeTypeHelper> typeHelperMap, final InputRow row, AggregatorFactory[] aggs) {
    try {
        List<String> parseExceptionMessages = new ArrayList<>();
        ByteArrayDataOutput out = ByteStreams.newDataOutput();
        // write timestamp
        out.writeLong(row.getTimestampFromEpoch());
        // writing all dimensions
        List<String> dimList = row.getDimensions();
        WritableUtils.writeVInt(out, dimList.size());
        for (String dim : dimList) {
            IndexSerdeTypeHelper typeHelper = typeHelperMap.get(dim);
            if (typeHelper == null) {
                typeHelper = STRING_HELPER;
            }
            writeString(dim, out);
            try {
                typeHelper.serialize(out, row.getRaw(dim));
            } catch (ParseException pe) {
                parseExceptionMessages.add(pe.getMessage());
            }
        }
        // writing all metrics
        Supplier<InputRow> supplier = () -> row;
        WritableUtils.writeVInt(out, aggs.length);
        for (AggregatorFactory aggFactory : aggs) {
            String k = aggFactory.getName();
            writeString(k, out);
            try (Aggregator agg = aggFactory.factorize(IncrementalIndex.makeColumnSelectorFactory(VirtualColumns.EMPTY, aggFactory, supplier, true))) {
                try {
                    agg.aggregate();
                } catch (ParseException e) {
                    // "aggregate" can throw ParseExceptions if a selector expects something but gets something else.
                    log.debug(e, "Encountered parse error, skipping aggregator[%s].", k);
                    parseExceptionMessages.add(e.getMessage());
                }
                final ColumnType type = aggFactory.getIntermediateType();
                if (agg.isNull()) {
                    out.writeByte(NullHandling.IS_NULL_BYTE);
                } else {
                    out.writeByte(NullHandling.IS_NOT_NULL_BYTE);
                    if (type.is(ValueType.FLOAT)) {
                        out.writeFloat(agg.getFloat());
                    } else if (type.is(ValueType.LONG)) {
                        WritableUtils.writeVLong(out, agg.getLong());
                    } else if (type.is(ValueType.DOUBLE)) {
                        out.writeDouble(agg.getDouble());
                    } else if (type.is(ValueType.COMPLEX)) {
                        Object val = agg.get();
                        ComplexMetricSerde serde = getComplexMetricSerde(type.getComplexTypeName());
                        writeBytes(serde.toBytes(val), out);
                    } else {
                        throw new IAE("Unable to serialize type[%s]", type.asTypeString());
                    }
                }
            }
        }
        return new SerializeResult(out.toByteArray(), parseExceptionMessages);
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
}