Examples with INTEGER com.facebook.presto.common.type.IntegerType.INTEGER used on opensource projects

Example 16 with INTEGER

use of com.facebook.presto.common.type.IntegerType.INTEGER in project presto by prestodb.

the class TestTranslateExpressions method testTranslateAggregationWithLambda.

@Test
public void testTranslateAggregationWithLambda() {
    PlanNode result = tester().assertThat(new TranslateExpressions(METADATA, new SqlParser()).aggregationRowExpressionRewriteRule()).on(p -> p.aggregation(builder -> builder.globalGrouping().addAggregation(variable("reduce_agg", INTEGER), new AggregationNode.Aggregation(new CallExpression("reduce_agg", REDUCE_AGG, INTEGER, ImmutableList.of(castToRowExpression(expression("input")), castToRowExpression(expression("0")), castToRowExpression(expression("(x,y) -> x*y")), castToRowExpression(expression("(a,b) -> a*b")))), Optional.of(castToRowExpression(expression("input > 10"))), Optional.empty(), false, Optional.empty())).source(p.values(p.variable("input", INTEGER))))).get();
    // TODO migrate this to RowExpressionMatcher
    AggregationNode.Aggregation translated = ((AggregationNode) result).getAggregations().get(variable("reduce_agg", INTEGER));
    assertEquals(translated, new AggregationNode.Aggregation(new CallExpression("reduce_agg", REDUCE_AGG, INTEGER, ImmutableList.of(variable("input", INTEGER), constant(0L, INTEGER), new LambdaDefinitionExpression(Optional.empty(), ImmutableList.of(INTEGER, INTEGER), ImmutableList.of("x", "y"), multiply(variable("x", INTEGER), variable("y", INTEGER))), new LambdaDefinitionExpression(Optional.empty(), ImmutableList.of(INTEGER, INTEGER), ImmutableList.of("a", "b"), multiply(variable("a", INTEGER), variable("b", INTEGER))))), Optional.of(greaterThan(variable("input", INTEGER), constant(10L, INTEGER))), Optional.empty(), false, Optional.empty()));
    assertFalse(isUntranslated(translated));
}

Example 17 with INTEGER

use of com.facebook.presto.common.type.IntegerType.INTEGER in project presto by prestodb.

the class TestOptimizedPartitionedOutputOperator method testPartitioned.

private void testPartitioned(List<Type> types, List<Page> pages, DataSize maxMemory, List<Integer> partitionChannel, HashGenerator hashGenerator) {
    TestingPartitionedOutputBuffer outputBuffer = createPartitionedOutputBuffer();
    PartitionFunction partitionFunction = new LocalPartitionGenerator(hashGenerator, PARTITION_COUNT);
    OptimizedPartitionedOutputOperator operator = createOptimizedPartitionedOutputOperator(types, partitionChannel, partitionFunction, outputBuffer, OptionalInt.empty(), maxMemory);
    Map<Integer, List<Page>> expectedPageList = new HashMap<>();
    for (Page page : pages) {
        Map<Integer, List<Integer>> positionsByPartition = new HashMap<>();
        for (int i = 0; i < page.getPositionCount(); i++) {
            int partitionNumber = partitionFunction.getPartition(page, i);
            positionsByPartition.computeIfAbsent(partitionNumber, k -> new ArrayList<>()).add(i);
        }
        for (Map.Entry<Integer, List<Integer>> entry : positionsByPartition.entrySet()) {
            if (!entry.getValue().isEmpty()) {
                expectedPageList.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(copyPositions(page, entry.getValue()));
            }
        }
        operator.addInput(page);
    }
    operator.finish();
    Map<Integer, Page> expectedPages = Maps.transformValues(expectedPageList, outputPages -> mergePages(types, outputPages));
    Map<Integer, Page> actualPages = Maps.transformValues(outputBuffer.getPages(), outputPages -> mergePages(types, outputPages));
    assertEquals(actualPages.size(), expectedPages.size());
    assertEquals(actualPages.keySet(), expectedPages.keySet());
    for (Map.Entry<Integer, Page> entry : expectedPages.entrySet()) {
        int key = entry.getKey();
        assertPageEquals(types, actualPages.get(key), entry.getValue());
    }
}

Example 18 with INTEGER

use of com.facebook.presto.common.type.IntegerType.INTEGER in project presto by prestodb.

the class ExtractSpatialJoins method tryCreateSpatialJoin.

private static Result tryCreateSpatialJoin(Context context, JoinNode joinNode, RowExpression filter, PlanNodeId nodeId, List<VariableReferenceExpression> outputVariables, CallExpression spatialFunction, Optional<RowExpression> radius, Metadata metadata, SplitManager splitManager, PageSourceManager pageSourceManager) {
    FunctionAndTypeManager functionAndTypeManager = metadata.getFunctionAndTypeManager();
    List<RowExpression> arguments = spatialFunction.getArguments();
    verify(arguments.size() == 2);
    RowExpression firstArgument = arguments.get(0);
    RowExpression secondArgument = arguments.get(1);
    // Currently, only inner joins are supported for spherical geometries.
    if (joinNode.getType() != INNER && isSphericalJoin(metadata, firstArgument, secondArgument)) {
        return Result.empty();
    }
    Set<VariableReferenceExpression> firstVariables = extractUnique(firstArgument);
    Set<VariableReferenceExpression> secondVariables = extractUnique(secondArgument);
    if (firstVariables.isEmpty() || secondVariables.isEmpty()) {
        return Result.empty();
    }
    // If either firstArgument or secondArgument is not a
    // VariableReferenceExpression, will replace the left/right join node
    // with a projection that adds the argument as a variable.
    Optional<VariableReferenceExpression> newFirstVariable = newGeometryVariable(context, firstArgument);
    Optional<VariableReferenceExpression> newSecondVariable = newGeometryVariable(context, secondArgument);
    PlanNode leftNode = joinNode.getLeft();
    PlanNode rightNode = joinNode.getRight();
    PlanNode newLeftNode;
    PlanNode newRightNode;
    // Check if the order of arguments of the spatial function matches the order of join sides
    int alignment = checkAlignment(joinNode, firstVariables, secondVariables);
    if (alignment > 0) {
        newLeftNode = newFirstVariable.map(variable -> addProjection(context, leftNode, variable, firstArgument)).orElse(leftNode);
        newRightNode = newSecondVariable.map(variable -> addProjection(context, rightNode, variable, secondArgument)).orElse(rightNode);
    } else if (alignment < 0) {
        newLeftNode = newSecondVariable.map(variable -> addProjection(context, leftNode, variable, secondArgument)).orElse(leftNode);
        newRightNode = newFirstVariable.map(variable -> addProjection(context, rightNode, variable, firstArgument)).orElse(rightNode);
    } else {
        return Result.empty();
    }
    RowExpression newFirstArgument = mapToExpression(newFirstVariable, firstArgument);
    RowExpression newSecondArgument = mapToExpression(newSecondVariable, secondArgument);
    // Implement partitioned spatial joins:
    // If the session parameter points to a valid spatial partitioning, use
    // that to assign to each probe and build rows the partitions that the
    // geometry intersects.  This is a projection that adds an array of ints
    // which is subsequently unnested.
    Optional<String> spatialPartitioningTableName = canPartitionSpatialJoin(joinNode) ? getSpatialPartitioningTableName(context.getSession()) : Optional.empty();
    Optional<KdbTree> kdbTree = spatialPartitioningTableName.map(tableName -> loadKdbTree(tableName, context.getSession(), metadata, splitManager, pageSourceManager));
    Optional<VariableReferenceExpression> leftPartitionVariable = Optional.empty();
    Optional<VariableReferenceExpression> rightPartitionVariable = Optional.empty();
    if (kdbTree.isPresent()) {
        leftPartitionVariable = Optional.of(context.getVariableAllocator().newVariable(newFirstArgument.getSourceLocation(), "pid", INTEGER));
        rightPartitionVariable = Optional.of(context.getVariableAllocator().newVariable(newSecondArgument.getSourceLocation(), "pid", INTEGER));
        if (alignment > 0) {
            newLeftNode = addPartitioningNodes(context, functionAndTypeManager, newLeftNode, leftPartitionVariable.get(), kdbTree.get(), newFirstArgument, Optional.empty());
            newRightNode = addPartitioningNodes(context, functionAndTypeManager, newRightNode, rightPartitionVariable.get(), kdbTree.get(), newSecondArgument, radius);
        } else {
            newLeftNode = addPartitioningNodes(context, functionAndTypeManager, newLeftNode, leftPartitionVariable.get(), kdbTree.get(), newSecondArgument, Optional.empty());
            newRightNode = addPartitioningNodes(context, functionAndTypeManager, newRightNode, rightPartitionVariable.get(), kdbTree.get(), newFirstArgument, radius);
        }
    }
    CallExpression newSpatialFunction = new CallExpression(spatialFunction.getSourceLocation(), spatialFunction.getDisplayName(), spatialFunction.getFunctionHandle(), spatialFunction.getType(), ImmutableList.of(newFirstArgument, newSecondArgument));
    RowExpression newFilter = RowExpressionNodeInliner.replaceExpression(filter, ImmutableMap.of(spatialFunction, newSpatialFunction));
    return Result.ofPlanNode(new SpatialJoinNode(joinNode.getSourceLocation(), nodeId, SpatialJoinNode.Type.fromJoinNodeType(joinNode.getType()), newLeftNode, newRightNode, outputVariables, newFilter, leftPartitionVariable, rightPartitionVariable, kdbTree.map(KdbTreeUtils::toJson)));
}

Example 19 with INTEGER

use of com.facebook.presto.common.type.IntegerType.INTEGER in project presto by prestodb.

the class TestSelectiveOrcReader method testVarchars.

@Test
public void testVarchars() throws Exception {
    Random random = new Random(0);
    tester.testRoundTripTypes(ImmutableList.of(VARCHAR, VARCHAR, VARCHAR), ImmutableList.of(newArrayList("abc", "def", null, "hij", "klm"), newArrayList(null, null, null, null, null), newArrayList("abc", "def", null, null, null)), toSubfieldFilters(ImmutableMap.of(0, stringIn(true, "abc", "def"), 1, stringIn(true, "10", "11"), 2, stringIn(true, "def", "abc"))));
    // direct and dictionary
    tester.testRoundTrip(VARCHAR, newArrayList(limit(cycle(ImmutableList.of("apple", "apple pie", "apple\uD835\uDC03", "apple\uFFFD")), NUM_ROWS)), stringIn(false, "apple", "apple pie"));
    // direct and dictionary materialized
    tester.testRoundTrip(VARCHAR, intsBetween(0, NUM_ROWS).stream().map(Object::toString).collect(toList()), stringIn(false, "10", "11"), stringIn(true, "10", "11"), stringBetween(false, "14", "10"));
    // direct & dictionary with filters
    tester.testRoundTripTypes(ImmutableList.of(VARCHAR, VARCHAR), ImmutableList.of(intsBetween(0, NUM_ROWS).stream().map(Object::toString).collect(toList()), newArrayList(limit(cycle(ImmutableList.of("A", "B", "C")), NUM_ROWS))), toSubfieldFilters(ImmutableMap.of(0, stringBetween(true, "16", "10"), 1, stringBetween(false, "B", "A"))));
    // stripe dictionary
    tester.testRoundTrip(VARCHAR, newArrayList(concat(ImmutableList.of("a"), nCopies(9999, "123"), ImmutableList.of("b"), nCopies(9999, "123"))));
    // empty sequence
    tester.testRoundTrip(VARCHAR, nCopies(NUM_ROWS, ""), stringEquals(false, ""));
    // copy of AbstractOrcTester::testDwrfInvalidCheckpointsForRowGroupDictionary
    List<Integer> values = newArrayList(limit(cycle(concat(ImmutableList.of(1), nCopies(9999, 123), ImmutableList.of(2), nCopies(9999, 123), ImmutableList.of(3), nCopies(9999, 123), nCopies(1_000_000, null))), 200_000));
    tester.assertRoundTrip(VARCHAR, newArrayList(values).stream().map(value -> value == null ? null : String.valueOf(value)).collect(toList()));
    // copy of AbstractOrcTester::testDwrfInvalidCheckpointsForStripeDictionary
    tester.testRoundTrip(VARCHAR, newArrayList(limit(cycle(ImmutableList.of(1, 3, 5, 7, 11, 13, 17)), 200_000)).stream().map(Object::toString).collect(toList()));
    // presentStream is null in some row groups & dictionary materialized
    Function<Integer, String> randomStrings = i -> String.valueOf(random.nextInt(NUM_ROWS));
    tester.testRoundTripTypes(ImmutableList.of(INTEGER, VARCHAR), ImmutableList.of(createList(NUM_ROWS, i -> random.nextInt(NUM_ROWS)), newArrayList(createList(NUM_ROWS, randomStrings))), toSubfieldFilters(ImmutableMap.of(0, BigintRange.of(10_000, 15_000, true))));
    // dataStream is null and lengths are 0
    tester.testRoundTrip(VARCHAR, newArrayList("", null), toSubfieldFilters(stringNotEquals(true, "")));
    tester.testRoundTrip(VARCHAR, newArrayList("", ""), toSubfieldFilters(stringLessThan(true, "")));
}

Example 20 with INTEGER

use of com.facebook.presto.common.type.IntegerType.INTEGER in project presto by prestodb.

the class TestSelectiveOrcReader method testArrayIndexOutOfBounds.

@Test
public void testArrayIndexOutOfBounds() throws Exception {
    Random random = new Random(0);
    // non-null arrays of varying sizes
    try {
        tester.testRoundTrip(arrayType(INTEGER), createList(NUM_ROWS, i -> randomIntegers(random.nextInt(10), random)), ImmutableList.of(ImmutableMap.of(new Subfield("c[2]"), IS_NULL)));
        fail("Expected 'Array subscript out of bounds' exception");
    } catch (InvalidFunctionArgumentException e) {
        assertTrue(e.getMessage().contains("Array subscript out of bounds"));
    }
    // non-null nested arrays of varying sizes
    try {
        tester.testRoundTrip(arrayType(arrayType(INTEGER)), createList(NUM_ROWS, i -> ImmutableList.of(randomIntegers(random.nextInt(5), random), randomIntegers(random.nextInt(5), random))), ImmutableList.of(ImmutableMap.of(new Subfield("c[2][3]"), IS_NULL)));
        fail("Expected 'Array subscript out of bounds' exception");
    } catch (InvalidFunctionArgumentException e) {
        assertTrue(e.getMessage().contains("Array subscript out of bounds"));
    }
    // empty arrays
    try {
        tester.testRoundTrip(arrayType(INTEGER), nCopies(NUM_ROWS, ImmutableList.of()), ImmutableList.of(ImmutableMap.of(new Subfield("c[2]"), IS_NULL)));
        fail("Expected 'Array subscript out of bounds' exception");
    } catch (InvalidFunctionArgumentException e) {
        assertTrue(e.getMessage().contains("Array subscript out of bounds"));
    }
    // empty nested arrays
    try {
        tester.testRoundTrip(arrayType(arrayType(INTEGER)), nCopies(NUM_ROWS, ImmutableList.of()), ImmutableList.of(ImmutableMap.of(new Subfield("c[2][3]"), IS_NULL)));
        fail("Expected 'Array subscript out of bounds' exception");
    } catch (InvalidFunctionArgumentException e) {
        assertTrue(e.getMessage().contains("Array subscript out of bounds"));
    }
}