Examples with Signature com.facebook.presto.metadata.Signature used on opensource projects

Example 66 with Signature

use of com.facebook.presto.metadata.Signature in project presto by prestodb.

the class TestHashAggregationOperator method testHashAggregationWithGlobals.

@Test(dataProvider = "hashEnabled")
public void testHashAggregationWithGlobals(boolean hashEnabled) throws Exception {
    MetadataManager metadata = MetadataManager.createTestMetadataManager();
    InternalAggregationFunction countVarcharColumn = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature("count", AGGREGATE, parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.VARCHAR)));
    InternalAggregationFunction countBooleanColumn = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature("count", AGGREGATE, parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.BOOLEAN)));
    InternalAggregationFunction maxVarcharColumn = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature("max", AGGREGATE, parseTypeSignature(StandardTypes.VARCHAR), parseTypeSignature(StandardTypes.VARCHAR)));
    Optional<Integer> groupIdChannel = Optional.of(1);
    List<Integer> groupByChannels = Ints.asList(1, 2);
    List<Integer> globalAggregationGroupIds = Ints.asList(42, 49);
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, groupByChannels, VARCHAR, VARCHAR, VARCHAR, BIGINT, BIGINT, BOOLEAN);
    List<Page> input = rowPagesBuilder.build();
    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(VARCHAR, BIGINT), groupByChannels, globalAggregationGroupIds, Step.SINGLE, ImmutableList.of(COUNT.bind(ImmutableList.of(0), Optional.empty()), LONG_SUM.bind(ImmutableList.of(4), Optional.empty()), LONG_AVERAGE.bind(ImmutableList.of(4), Optional.empty()), maxVarcharColumn.bind(ImmutableList.of(2), Optional.empty()), countVarcharColumn.bind(ImmutableList.of(0), Optional.empty()), countBooleanColumn.bind(ImmutableList.of(5), Optional.empty())), rowPagesBuilder.getHashChannel(), groupIdChannel, 100_000, new DataSize(16, MEGABYTE), joinCompiler);
    MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, BIGINT, BIGINT, BIGINT, DOUBLE, VARCHAR, BIGINT, BIGINT).row(null, 42L, 0L, null, null, null, 0L, 0L).row(null, 49L, 0L, null, null, null, 0L, 0L).build();
    assertOperatorEqualsIgnoreOrder(operatorFactory, driverContext, input, expected, hashEnabled, Optional.of(groupByChannels.size()));
}

Example 67 with Signature

use of com.facebook.presto.metadata.Signature in project presto by prestodb.

the class TestHashAggregationOperator method testSpillerFailure.

@Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".* Failed to spill")
public void testSpillerFailure() {
    MetadataManager metadata = MetadataManager.createTestMetadataManager();
    InternalAggregationFunction maxVarcharColumn = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature("max", AGGREGATE, parseTypeSignature(StandardTypes.VARCHAR), parseTypeSignature(StandardTypes.VARCHAR)));
    List<Integer> hashChannels = Ints.asList(1);
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(false, hashChannels, VARCHAR, BIGINT, VARCHAR, BIGINT);
    List<Page> input = rowPagesBuilder.addSequencePage(10, 100, 0, 100, 0).addSequencePage(10, 100, 0, 200, 0).addSequencePage(10, 100, 0, 300, 0).build();
    DriverContext driverContext = createTaskContext(executor, TEST_SESSION, new DataSize(10, Unit.BYTE)).addPipelineContext(0, true, true).addDriverContext();
    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT), hashChannels, ImmutableList.of(), Step.SINGLE, ImmutableList.of(COUNT.bind(ImmutableList.of(0), Optional.empty()), LONG_SUM.bind(ImmutableList.of(3), Optional.empty()), LONG_AVERAGE.bind(ImmutableList.of(3), Optional.empty()), maxVarcharColumn.bind(ImmutableList.of(2), Optional.empty())), rowPagesBuilder.getHashChannel(), Optional.empty(), 100_000, new DataSize(16, MEGABYTE), true, succinctBytes(8), succinctBytes(Integer.MAX_VALUE), new FailingSpillerFactory(), joinCompiler);
    toPages(operatorFactory, driverContext, input);
}

Example 68 with Signature

use of com.facebook.presto.metadata.Signature in project presto by prestodb.

the class SwitchCodeGenerator method generateExpression.

@Override
public BytecodeNode generateExpression(Signature signature, BytecodeGeneratorContext generatorContext, Type returnType, List<RowExpression> arguments) {
    // TODO: compile as
    /*
            hashCode = hashCode(<value>)

            // all constant expressions before a non-constant
            switch (hashCode) {
                case ...:
                    if (<value> == <constant1>) {
                       ...
                    }
                    else if (<value> == <constant2>) {
                       ...
                    }
                    else if (...) {
                    }
                case ...:
                    ...
            }

            if (<value> == <non-constant1>) {
                ...
            }
            else if (<value> == <non-constant2>) {
                ...
            }
            ...

            // repeat with next sequence of constant expressions
         */
    Scope scope = generatorContext.getScope();
    // process value, else, and all when clauses
    RowExpression value = arguments.get(0);
    BytecodeNode valueBytecode = generatorContext.generate(value);
    BytecodeNode elseValue;
    List<RowExpression> whenClauses;
    RowExpression last = arguments.get(arguments.size() - 1);
    if (last instanceof CallExpression && ((CallExpression) last).getSignature().getName().equals("WHEN")) {
        whenClauses = arguments.subList(1, arguments.size());
        elseValue = new BytecodeBlock().append(generatorContext.wasNull().set(constantTrue())).pushJavaDefault(returnType.getJavaType());
    } else {
        whenClauses = arguments.subList(1, arguments.size() - 1);
        elseValue = generatorContext.generate(last);
    }
    // determine the type of the value and result
    Class<?> valueType = value.getType().getJavaType();
    // evaluate the value and store it in a variable
    LabelNode nullValue = new LabelNode("nullCondition");
    Variable tempVariable = scope.createTempVariable(valueType);
    BytecodeBlock block = new BytecodeBlock().append(valueBytecode).append(BytecodeUtils.ifWasNullClearPopAndGoto(scope, nullValue, void.class, valueType)).putVariable(tempVariable);
    BytecodeNode getTempVariableNode = VariableInstruction.loadVariable(tempVariable);
    // build the statements
    elseValue = new BytecodeBlock().visitLabel(nullValue).append(elseValue);
    // reverse list because current if statement builder doesn't support if/else so we need to build the if statements bottom up
    for (RowExpression clause : Lists.reverse(whenClauses)) {
        Preconditions.checkArgument(clause instanceof CallExpression && ((CallExpression) clause).getSignature().getName().equals("WHEN"));
        RowExpression operand = ((CallExpression) clause).getArguments().get(0);
        RowExpression result = ((CallExpression) clause).getArguments().get(1);
        // call equals(value, operand)
        Signature equalsFunction = generatorContext.getRegistry().resolveOperator(OperatorType.EQUAL, ImmutableList.of(value.getType(), operand.getType()));
        // TODO: what if operand is null? It seems that the call will return "null" (which is cleared below)
        // and the code only does the right thing because the value in the stack for that scenario is
        // Java's default for boolean == false
        // This code should probably be checking for wasNull after the call and "failing" the equality
        // check if wasNull is true
        BytecodeNode equalsCall = generatorContext.generateCall(equalsFunction.getName(), generatorContext.getRegistry().getScalarFunctionImplementation(equalsFunction), ImmutableList.of(generatorContext.generate(operand), getTempVariableNode));
        BytecodeBlock condition = new BytecodeBlock().append(equalsCall).append(generatorContext.wasNull().set(constantFalse()));
        elseValue = new IfStatement("when").condition(condition).ifTrue(generatorContext.generate(result)).ifFalse(elseValue);
    }
    return block.append(elseValue);
}

Example 69 with Signature

use of com.facebook.presto.metadata.Signature in project presto by prestodb.

the class LiteralInterpreter method toExpression.

public static Expression toExpression(Object object, Type type) {
    requireNonNull(type, "type is null");
    if (object instanceof Expression) {
        return (Expression) object;
    }
    if (object == null) {
        if (type.equals(UNKNOWN)) {
            return new NullLiteral();
        }
        return new Cast(new NullLiteral(), type.getTypeSignature().toString(), false, true);
    }
    if (type.equals(INTEGER)) {
        return new LongLiteral(object.toString());
    }
    if (type.equals(BIGINT)) {
        LongLiteral expression = new LongLiteral(object.toString());
        if (expression.getValue() >= Integer.MIN_VALUE && expression.getValue() <= Integer.MAX_VALUE) {
            return new GenericLiteral("BIGINT", object.toString());
        }
        return new LongLiteral(object.toString());
    }
    checkArgument(Primitives.wrap(type.getJavaType()).isInstance(object), "object.getClass (%s) and type.getJavaType (%s) do not agree", object.getClass(), type.getJavaType());
    if (type.equals(DOUBLE)) {
        Double value = (Double) object;
        // When changing this, don't forget about similar code for REAL below
        if (value.isNaN()) {
            return new FunctionCall(QualifiedName.of("nan"), ImmutableList.of());
        } else if (value.equals(Double.NEGATIVE_INFINITY)) {
            return ArithmeticUnaryExpression.negative(new FunctionCall(QualifiedName.of("infinity"), ImmutableList.of()));
        } else if (value.equals(Double.POSITIVE_INFINITY)) {
            return new FunctionCall(QualifiedName.of("infinity"), ImmutableList.of());
        } else {
            return new DoubleLiteral(object.toString());
        }
    }
    if (type.equals(REAL)) {
        Float value = intBitsToFloat(((Long) object).intValue());
        // WARNING for ORC predicate code as above (for double)
        if (value.isNaN()) {
            return new Cast(new FunctionCall(QualifiedName.of("nan"), ImmutableList.of()), StandardTypes.REAL);
        } else if (value.equals(Float.NEGATIVE_INFINITY)) {
            return ArithmeticUnaryExpression.negative(new Cast(new FunctionCall(QualifiedName.of("infinity"), ImmutableList.of()), StandardTypes.REAL));
        } else if (value.equals(Float.POSITIVE_INFINITY)) {
            return new Cast(new FunctionCall(QualifiedName.of("infinity"), ImmutableList.of()), StandardTypes.REAL);
        } else {
            return new GenericLiteral("REAL", value.toString());
        }
    }
    if (type instanceof VarcharType) {
        if (object instanceof String) {
            object = Slices.utf8Slice((String) object);
        }
        if (object instanceof Slice) {
            Slice value = (Slice) object;
            int length = SliceUtf8.countCodePoints(value);
            if (length == ((VarcharType) type).getLength()) {
                return new StringLiteral(value.toStringUtf8());
            }
            return new Cast(new StringLiteral(value.toStringUtf8()), type.getDisplayName(), false, true);
        }
        throw new IllegalArgumentException("object must be instance of Slice or String when type is VARCHAR");
    }
    if (type.equals(BOOLEAN)) {
        return new BooleanLiteral(object.toString());
    }
    if (object instanceof Block) {
        SliceOutput output = new DynamicSliceOutput(((Block) object).getSizeInBytes());
        BlockSerdeUtil.writeBlock(output, (Block) object);
        object = output.slice();
    // This if condition will evaluate to true: object instanceof Slice && !type.equals(VARCHAR)
    }
    if (object instanceof Slice) {
        // HACK: we need to serialize VARBINARY in a format that can be embedded in an expression to be
        // able to encode it in the plan that gets sent to workers.
        // We do this by transforming the in-memory varbinary into a call to from_base64(<base64-encoded value>)
        FunctionCall fromBase64 = new FunctionCall(QualifiedName.of("from_base64"), ImmutableList.of(new StringLiteral(VarbinaryFunctions.toBase64((Slice) object).toStringUtf8())));
        Signature signature = FunctionRegistry.getMagicLiteralFunctionSignature(type);
        return new FunctionCall(QualifiedName.of(signature.getName()), ImmutableList.of(fromBase64));
    }
    Signature signature = FunctionRegistry.getMagicLiteralFunctionSignature(type);
    Expression rawLiteral = toExpression(object, FunctionRegistry.typeForMagicLiteral(type));
    return new FunctionCall(QualifiedName.of(signature.getName()), ImmutableList.of(rawLiteral));
}

Example 70 with Signature

use of com.facebook.presto.metadata.Signature in project presto by prestodb.

the class TestEvaluateClassifierPredictions method testEvaluateClassifierPredictions.

@Test
public void testEvaluateClassifierPredictions() throws Exception {
    metadata.addFunctions(extractFunctions(new MLPlugin().getFunctions()));
    InternalAggregationFunction aggregation = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature("evaluate_classifier_predictions", AGGREGATE, parseTypeSignature(StandardTypes.VARCHAR), parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.BIGINT)));
    Accumulator accumulator = aggregation.bind(ImmutableList.of(0, 1), Optional.empty()).createAccumulator();
    accumulator.addInput(getPage());
    BlockBuilder finalOut = accumulator.getFinalType().createBlockBuilder(new BlockBuilderStatus(), 1);
    accumulator.evaluateFinal(finalOut);
    Block block = finalOut.build();
    String output = VARCHAR.getSlice(block, 0).toStringUtf8();
    List<String> parts = ImmutableList.copyOf(Splitter.on('\n').omitEmptyStrings().split(output));
    assertEquals(parts.size(), 7, output);
    assertEquals(parts.get(0), "Accuracy: 1/2 (50.00%)");
}