Examples with TypedSet com.facebook.presto.operator.aggregation.TypedSet used on opensource projects

Example 6 with TypedSet

use of com.facebook.presto.operator.aggregation.TypedSet in project presto by prestodb.

the class MapToMapCast method toMap.

@TypeParameter("FK")
@TypeParameter("FV")
@TypeParameter("TK")
@TypeParameter("TV")
@SqlType("map(TK,TV)")
public static Block toMap(@OperatorDependency(operator = EQUAL, returnType = StandardTypes.BOOLEAN, argumentTypes = { "TK", "TK" }) MethodHandle toKeyEqualsFunction, @OperatorDependency(operator = CAST, returnType = "TK", argumentTypes = { "FK" }) MethodHandle keyCastFunction, @OperatorDependency(operator = CAST, returnType = "TV", argumentTypes = { "FV" }) MethodHandle valueCastFunction, @TypeParameter("FK") Type fromKeyType, @TypeParameter("FV") Type fromValueType, @TypeParameter("TK") Type toKeyType, @TypeParameter("TV") Type toValueType, ConnectorSession session, @SqlType("map(FK,FV)") Block fromMap) {
    // loop over all the parameter types and bind ConnectorSession if needed
    // TODO: binding `ConnectorSession` should be done in function invocation framework
    Class<?>[] parameterTypes = keyCastFunction.type().parameterArray();
    for (int i = 0; i < parameterTypes.length; i++) {
        if (parameterTypes[i] == ConnectorSession.class) {
            keyCastFunction = MethodHandles.insertArguments(keyCastFunction, i, session);
            break;
        }
    }
    parameterTypes = valueCastFunction.type().parameterArray();
    for (int i = 0; i < parameterTypes.length; i++) {
        if (parameterTypes[i] == ConnectorSession.class) {
            valueCastFunction = MethodHandles.insertArguments(valueCastFunction, i, session);
            break;
        }
    }
    TypedSet typedSet = new TypedSet(toKeyType, fromMap.getPositionCount() / 2);
    BlockBuilder keyBlockBuilder = toKeyType.createBlockBuilder(new BlockBuilderStatus(), fromMap.getPositionCount() / 2);
    for (int i = 0; i < fromMap.getPositionCount(); i += 2) {
        Object fromKey = readNativeValue(fromKeyType, fromMap, i);
        try {
            Object toKey = keyCastFunction.invoke(fromKey);
            if (toKey == null) {
                throw new PrestoException(StandardErrorCode.INVALID_CAST_ARGUMENT, "map key is null");
            }
            writeNativeValue(toKeyType, keyBlockBuilder, toKey);
        } catch (Throwable t) {
            Throwables.propagateIfInstanceOf(t, Error.class);
            Throwables.propagateIfInstanceOf(t, PrestoException.class);
            throw new PrestoException(StandardErrorCode.GENERIC_INTERNAL_ERROR, t);
        }
    }
    Block keyBlock = keyBlockBuilder.build();
    BlockBuilder blockBuilder = new InterleavedBlockBuilder(ImmutableList.of(toKeyType, toValueType), new BlockBuilderStatus(), fromMap.getPositionCount());
    for (int i = 0; i < fromMap.getPositionCount(); i += 2) {
        if (!typedSet.contains(keyBlock, i / 2)) {
            typedSet.add(keyBlock, i / 2);
            toKeyType.appendTo(keyBlock, i / 2, blockBuilder);
            if (fromMap.isNull(i + 1)) {
                blockBuilder.appendNull();
                continue;
            }
            Object fromValue = readNativeValue(fromValueType, fromMap, i + 1);
            try {
                Object toValue = valueCastFunction.invoke(fromValue);
                writeNativeValue(toValueType, blockBuilder, toValue);
            } catch (Throwable t) {
                Throwables.propagateIfInstanceOf(t, Error.class);
                Throwables.propagateIfInstanceOf(t, PrestoException.class);
                throw new PrestoException(StandardErrorCode.GENERIC_INTERNAL_ERROR, t);
            }
        } else {
            // if there are duplicated keys, fail it!
            throw new PrestoException(StandardErrorCode.INVALID_CAST_ARGUMENT, "duplicate keys");
        }
    }
    return blockBuilder.build();
}

Example 7 with TypedSet

use of com.facebook.presto.operator.aggregation.TypedSet in project presto by prestodb.

the class MapConcatFunction method mapConcat.

@UsedByGeneratedCode
public static Block mapConcat(Type keyType, Type valueType, Object state, Block[] maps) {
    int entries = 0;
    int lastMapIndex = maps.length - 1;
    int firstMapIndex = lastMapIndex;
    for (int i = 0; i < maps.length; i++) {
        entries += maps[i].getPositionCount();
        if (maps[i].getPositionCount() > 0) {
            lastMapIndex = i;
            firstMapIndex = min(firstMapIndex, i);
        }
    }
    if (lastMapIndex == firstMapIndex) {
        return maps[lastMapIndex];
    }
    PageBuilder pageBuilder = (PageBuilder) state;
    if (pageBuilder.isFull()) {
        pageBuilder.reset();
    }
    // TODO: we should move TypedSet into user state as well
    TypedSet typedSet = new TypedSet(keyType, entries / 2);
    BlockBuilder keyBlockBuilder = pageBuilder.getBlockBuilder(0);
    BlockBuilder valueBlockBuilder = pageBuilder.getBlockBuilder(1);
    // the last map
    Block map = maps[lastMapIndex];
    int total = 0;
    for (int i = 0; i < map.getPositionCount(); i += 2) {
        typedSet.add(map, i);
        keyType.appendTo(map, i, keyBlockBuilder);
        valueType.appendTo(map, i + 1, valueBlockBuilder);
        total++;
    }
    // the map between the last and the first
    for (int idx = lastMapIndex - 1; idx > firstMapIndex; idx--) {
        map = maps[idx];
        for (int i = 0; i < map.getPositionCount(); i += 2) {
            if (!typedSet.contains(map, i)) {
                typedSet.add(map, i);
                keyType.appendTo(map, i, keyBlockBuilder);
                valueType.appendTo(map, i + 1, valueBlockBuilder);
                total++;
            }
        }
    }
    // the first map
    map = maps[firstMapIndex];
    for (int i = 0; i < map.getPositionCount(); i += 2) {
        if (!typedSet.contains(map, i)) {
            keyType.appendTo(map, i, keyBlockBuilder);
            valueType.appendTo(map, i + 1, valueBlockBuilder);
            total++;
        }
    }
    pageBuilder.declarePositions(total);
    Block[] blocks = new Block[2];
    blocks[0] = keyBlockBuilder.getRegion(keyBlockBuilder.getPositionCount() - total, total);
    blocks[1] = valueBlockBuilder.getRegion(valueBlockBuilder.getPositionCount() - total, total);
    return new InterleavedBlock(blocks);
}

Example 8 with TypedSet

use of com.facebook.presto.operator.aggregation.TypedSet in project presto by prestodb.

the class DynamicFilterSourceOperator method addInput.

@Override
public void addInput(Page page) {
    verify(!finished, "DynamicFilterSourceOperator: addInput() shouldn't not be called after finish()");
    current = page;
    if (valueSets == null) {
        // the exact predicate became too large.
        if (minValues == null) {
            // there are too many rows to collect min/max range
            return;
        }
        minMaxCollectionLimit -= page.getPositionCount();
        if (minMaxCollectionLimit < 0) {
            handleMinMaxCollectionLimitExceeded();
            return;
        }
        // the predicate became too large, record only min and max values for each orderable channel
        for (Integer channelIndex : minMaxChannels) {
            Block block = page.getBlock(channels.get(channelIndex).index);
            updateMinMaxValues(block, channelIndex);
        }
        return;
    }
    minMaxCollectionLimit -= page.getPositionCount();
    // TODO: we should account for the memory used for collecting build-side values using MemoryContext
    long filterSizeInBytes = 0;
    int filterPositionsCount = 0;
    // Collect only the columns which are relevant for the JOIN.
    for (int channelIndex = 0; channelIndex < channels.size(); ++channelIndex) {
        Block block = page.getBlock(channels.get(channelIndex).getIndex());
        TypedSet valueSet = valueSets[channelIndex];
        for (int position = 0; position < block.getPositionCount(); ++position) {
            valueSet.add(block, position);
        }
        filterSizeInBytes += valueSet.getRetainedSizeInBytes();
        filterPositionsCount += valueSet.size();
    }
    if (filterPositionsCount > maxFilterPositionsCount || filterSizeInBytes > maxFilterSizeInBytes) {
        // The whole filter (summed over all columns) contains too much values or exceeds maxFilterSizeInBytes.
        handleTooLargePredicate();
    }
}

Example 9 with TypedSet

use of com.facebook.presto.operator.aggregation.TypedSet in project presto by prestodb.

the class MathFunctions method mapDotProduct.

private static double mapDotProduct(Block leftMap, Block rightMap) {
    TypedSet rightMapKeys = new TypedSet(VARCHAR, rightMap.getPositionCount(), "cosine_similarity");
    for (int i = 0; i < rightMap.getPositionCount(); i += 2) {
        rightMapKeys.add(rightMap, i);
    }
    double result = 0.0;
    for (int i = 0; i < leftMap.getPositionCount(); i += 2) {
        int position = rightMapKeys.positionOf(leftMap, i);
        if (position != -1) {
            result += DOUBLE.getDouble(leftMap, i + 1) * DOUBLE.getDouble(rightMap, 2 * position + 1);
        }
    }
    return result;
}

Example 10 with TypedSet

use of com.facebook.presto.operator.aggregation.TypedSet in project presto by prestodb.

the class MapToMapCast method mapCast.

@UsedByGeneratedCode
public static Block mapCast(MethodHandle keyProcessFunction, MethodHandle valueProcessFunction, Type toMapType, SqlFunctionProperties properties, Block fromMap) {
    checkState(toMapType.getTypeParameters().size() == 2, "Expect two type parameters for toMapType");
    Type toKeyType = toMapType.getTypeParameters().get(0);
    TypedSet typedSet = new TypedSet(toKeyType, fromMap.getPositionCount() / 2, "map-to-map cast");
    BlockBuilder keyBlockBuilder = toKeyType.createBlockBuilder(null, fromMap.getPositionCount() / 2);
    for (int i = 0; i < fromMap.getPositionCount(); i += 2) {
        try {
            keyProcessFunction.invokeExact(fromMap, i, properties, keyBlockBuilder);
        } catch (Throwable t) {
            throw internalError(t);
        }
    }
    Block keyBlock = keyBlockBuilder.build();
    BlockBuilder mapBlockBuilder = toMapType.createBlockBuilder(null, 1);
    BlockBuilder blockBuilder = mapBlockBuilder.beginBlockEntry();
    for (int i = 0; i < fromMap.getPositionCount(); i += 2) {
        if (!typedSet.contains(keyBlock, i / 2)) {
            typedSet.add(keyBlock, i / 2);
            toKeyType.appendTo(keyBlock, i / 2, blockBuilder);
            if (fromMap.isNull(i + 1)) {
                blockBuilder.appendNull();
                continue;
            }
            try {
                valueProcessFunction.invokeExact(fromMap, i + 1, properties, blockBuilder);
            } catch (Throwable t) {
                throw internalError(t);
            }
        } else {
            // if there are duplicated keys, fail it!
            throw new PrestoException(INVALID_CAST_ARGUMENT, "duplicate keys");
        }
    }
    mapBlockBuilder.closeEntry();
    return (Block) toMapType.getObject(mapBlockBuilder, mapBlockBuilder.getPositionCount() - 1);
}