Examples with Pair org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.Pair used on opensource projects

Example 81 with Pair

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.Pair in project beam by apache.

the class BeamIOPushDownRule method onMatch.

// ~ Methods ----------------------------------------------------------------
@Override
public void onMatch(RelOptRuleCall call) {
    final BeamIOSourceRel ioSourceRel = call.rel(1);
    final BeamSqlTable beamSqlTable = ioSourceRel.getBeamSqlTable();
    if (ioSourceRel instanceof BeamPushDownIOSourceRel) {
        return;
    }
    // Nested rows are not supported at the moment
    for (RelDataTypeField field : ioSourceRel.getRowType().getFieldList()) {
        if (field.getType() instanceof RelRecordType) {
            return;
        }
    }
    final Calc calc = call.rel(0);
    final RexProgram program = calc.getProgram();
    final Pair<ImmutableList<RexNode>, ImmutableList<RexNode>> projectFilter = program.split();
    final RelDataType calcInputRowType = program.getInputRowType();
    // When predicate push-down is not supported - all filters are unsupported.
    final BeamSqlTableFilter tableFilter = beamSqlTable.constructFilter(projectFilter.right);
    if (!beamSqlTable.supportsProjects().isSupported() && tableFilter instanceof DefaultTableFilter) {
        // Either project or filter push-down must be supported by the IO.
        return;
    }
    Set<String> usedFields = new LinkedHashSet<>();
    if (!(tableFilter instanceof DefaultTableFilter) && !beamSqlTable.supportsProjects().isSupported()) {
        // When applying standalone filter push-down all fields must be project by an IO.
        // With a single exception: Calc projects all fields (in the same order) and does nothing
        // else.
        usedFields.addAll(calcInputRowType.getFieldNames());
    } else {
        // Find all input refs used by projects
        for (RexNode project : projectFilter.left) {
            findUtilizedInputRefs(calcInputRowType, project, usedFields);
        }
        // Find all input refs used by filters
        for (RexNode filter : tableFilter.getNotSupported()) {
            findUtilizedInputRefs(calcInputRowType, filter, usedFields);
        }
    }
    if (usedFields.isEmpty()) {
        // No need to do push-down for queries like this: "select UPPER('hello')".
        return;
    }
    // IO only projects fields utilized by a calc.
    if (tableFilter.getNotSupported().containsAll(projectFilter.right) && usedFields.containsAll(ioSourceRel.getRowType().getFieldNames())) {
        return;
    }
    FieldAccessDescriptor resolved = FieldAccessDescriptor.withFieldNames(usedFields);
    resolved = resolved.resolve(beamSqlTable.getSchema());
    if (canDropCalc(program, beamSqlTable.supportsProjects(), tableFilter)) {
        call.transformTo(ioSourceRel.createPushDownRel(calc.getRowType(), resolved.getFieldsAccessed().stream().map(FieldDescriptor::getFieldName).collect(Collectors.toList()), tableFilter));
        return;
    }
    // IO only projects fields utilised by a calc.
    if (tableFilter.getNotSupported().equals(projectFilter.right) && usedFields.containsAll(ioSourceRel.getRowType().getFieldNames())) {
        return;
    }
    RelNode result = constructNodesWithPushDown(resolved, call.builder(), ioSourceRel, tableFilter, calc.getRowType(), projectFilter.left);
    if (tableFilter.getNotSupported().size() <= projectFilter.right.size() || usedFields.size() < calcInputRowType.getFieldCount()) {
        // Smaller Calc programs are indisputably better, as well as IOs with less projected fields.
        // We can consider something with the same number of filters.
        call.transformTo(result);
    }
}

Example 82 with Pair

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.Pair in project beam by apache.

the class BeamJoinRel method extractJoinRexNodes.

static List<Pair<RexNode, RexNode>> extractJoinRexNodes(RexNode condition) {
    // or it's a JOIN ON false because: condition == false
    if (condition instanceof RexLiteral) {
        throw new UnsupportedOperationException("CROSS JOIN, JOIN ON FALSE is not supported!");
    }
    RexCall call = (RexCall) condition;
    List<Pair<RexNode, RexNode>> pairs = new ArrayList<>();
    if ("AND".equals(call.getOperator().getName())) {
        List<RexNode> operands = call.getOperands();
        for (RexNode rexNode : operands) {
            Pair<RexNode, RexNode> pair = extractJoinPairOfRexNodes((RexCall) rexNode);
            pairs.add(pair);
        }
    } else if ("=".equals(call.getOperator().getName())) {
        pairs.add(extractJoinPairOfRexNodes(call));
    } else {
        throw new UnsupportedOperationException("Operator " + call.getOperator().getName() + " is not supported in join condition");
    }
    return pairs;
}

Example 83 with Pair

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.Pair in project hazelcast by hazelcast.

the class HazelcastSqlToRelConverter method constructComparisons.

/**
 * Constructs comparisons between
 * left-hand operand (as a rule, SqlIdentifier) and right-hand list.
 */
private List<RexNode> constructComparisons(Blackboard bb, List<RexNode> leftKeys, SqlNodeList valuesList) {
    final List<RexNode> comparisons = new ArrayList<>();
    for (SqlNode rightValues : valuesList) {
        RexNode rexComparison;
        final SqlOperator comparisonOp = SqlStdOperatorTable.EQUALS;
        if (leftKeys.size() == 1) {
            rexComparison = rexBuilder.makeCall(comparisonOp, leftKeys.get(0), ensureSqlType(leftKeys.get(0).getType(), bb.convertExpression(rightValues)));
        } else {
            assert rightValues instanceof SqlCall;
            final SqlBasicCall basicCall = (SqlBasicCall) rightValues;
            assert basicCall.getOperator() instanceof SqlRowOperator && basicCall.operandCount() == leftKeys.size();
            rexComparison = RexUtil.composeConjunction(rexBuilder, Pair.zip(leftKeys, basicCall.getOperandList()).stream().map(pair -> rexBuilder.makeCall(comparisonOp, pair.left, ensureSqlType(pair.left.getType(), bb.convertExpression(pair.right)))).collect(Collectors.toList()));
        }
        comparisons.add(rexComparison);
    }
    return comparisons;
}

Example 84 with Pair

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.Pair in project beam by apache.

the class IOPushDownRuleTest method testFindUtilisedInputRefs.

@Test
public void testFindUtilisedInputRefs() {
    String sqlQuery = "select id+10 from TEST where name='one'";
    BeamRelNode basicRel = sqlEnv.parseQuery(sqlQuery);
    assertThat(basicRel, instanceOf(Calc.class));
    Calc calc = (Calc) basicRel;
    final Pair<ImmutableList<RexNode>, ImmutableList<RexNode>> projectFilter = calc.getProgram().split();
    final ImmutableList<RexNode> projects = projectFilter.left;
    final ImmutableList<RexNode> filters = projectFilter.right;
    Set<String> usedFields = new HashSet<>();
    BeamIOPushDownRule.INSTANCE.findUtilizedInputRefs(calc.getProgram().getInputRowType(), projects.get(0), usedFields);
    assertThat(usedFields, containsInAnyOrder("id"));
    BeamIOPushDownRule.INSTANCE.findUtilizedInputRefs(calc.getProgram().getInputRowType(), filters.get(0), usedFields);
    assertThat(usedFields, containsInAnyOrder("id", "name"));
}

Example 85 with Pair

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.Pair in project beam by apache.

the class IOPushDownRuleTest method testReMapRexNodeToNewInputs.

@Test
public void testReMapRexNodeToNewInputs() {
    String sqlQuery = "select id+10 from TEST where name='one'";
    BeamRelNode basicRel = sqlEnv.parseQuery(sqlQuery);
    assertThat(basicRel, instanceOf(Calc.class));
    Calc calc = (Calc) basicRel;
    final Pair<ImmutableList<RexNode>, ImmutableList<RexNode>> projectFilter = calc.getProgram().split();
    final ImmutableList<RexNode> projects = projectFilter.left;
    final ImmutableList<RexNode> filters = projectFilter.right;
    List<Integer> mapping = ImmutableList.of(1, 2);
    RexNode newProject = BeamIOPushDownRule.INSTANCE.reMapRexNodeToNewInputs(projects.get(0), mapping);
    assertThat(newProject.toString(), equalTo("+($0, 10)"));
    RexNode newFilter = BeamIOPushDownRule.INSTANCE.reMapRexNodeToNewInputs(filters.get(0), mapping);
    assertThat(newFilter.toString(), equalTo("=($1, 'one')"));
}