Examples with RexTableInputRef org.apache.calcite.rex.RexTableInputRef used on opensource projects

Example 11 with RexTableInputRef

use of org.apache.calcite.rex.RexTableInputRef in project calcite by apache.

the class RelMetadataTest method testExpressionLineageMultiUnion.

@Test
public void testExpressionLineageMultiUnion() {
    // empno is column 0 in catalog.sales.emp
    // sal is column 5 in catalog.sales.emp
    final RelNode rel = convertSql("select a.empno + b.sal from \n" + " (select empno, ename from emp,dept) a join " + " (select * from emp union all select * from emp) b \n" + " on a.empno = b.empno \n" + " where b.deptno = 10");
    final RelMetadataQuery mq = RelMetadataQuery.instance();
    final RexNode ref = RexInputRef.of(0, rel.getRowType().getFieldList());
    final Set<RexNode> r = mq.getExpressionLineage(rel, ref);
    // With the union, we should get two origins
    // The first one should be the same one: join
    // The second should come from each union input
    final Set<List<String>> set = new HashSet<>();
    assertThat(r.size(), is(2));
    for (RexNode result : r) {
        assertThat(result.getKind(), is(SqlKind.PLUS));
        final RexCall call = (RexCall) result;
        assertThat(call.getOperands().size(), is(2));
        final RexTableInputRef inputRef1 = (RexTableInputRef) call.getOperands().get(0);
        assertTrue(inputRef1.getQualifiedName().equals(EMP_QNAME));
        // Add join alpha to set
        set.add(inputRef1.getQualifiedName());
        assertThat(inputRef1.getIndex(), is(0));
        final RexTableInputRef inputRef2 = (RexTableInputRef) call.getOperands().get(1);
        assertTrue(inputRef2.getQualifiedName().equals(EMP_QNAME));
        assertThat(inputRef2.getIndex(), is(5));
        assertThat(inputRef1.getIdentifier(), not(inputRef2.getIdentifier()));
    }
    assertThat(set.size(), is(1));
}

Example 12 with RexTableInputRef

use of org.apache.calcite.rex.RexTableInputRef in project calcite by apache.

the class RelMetadataTest method testExpressionLineageTwoColumnsSwapped.

@Test
public void testExpressionLineageTwoColumnsSwapped() {
    // deptno is column 7 in catalog.sales.emp
    // mgr is column 3 in catalog.sales.emp
    final RelNode rel = convertSql("select deptno, mgr from emp");
    final RelMetadataQuery mq = RelMetadataQuery.instance();
    final RexNode ref1 = RexInputRef.of(0, rel.getRowType().getFieldList());
    final Set<RexNode> r1 = mq.getExpressionLineage(rel, ref1);
    assertThat(r1.size(), is(1));
    final RexTableInputRef result1 = (RexTableInputRef) r1.iterator().next();
    assertTrue(result1.getQualifiedName().equals(EMP_QNAME));
    assertThat(result1.getIndex(), is(7));
    final RexNode ref2 = RexInputRef.of(1, rel.getRowType().getFieldList());
    final Set<RexNode> r2 = mq.getExpressionLineage(rel, ref2);
    assertThat(r2.size(), is(1));
    final RexTableInputRef result2 = (RexTableInputRef) r2.iterator().next();
    assertTrue(result2.getQualifiedName().equals(EMP_QNAME));
    assertThat(result2.getIndex(), is(3));
    assertThat(result1.getIdentifier(), is(result2.getIdentifier()));
}

Example 13 with RexTableInputRef

use of org.apache.calcite.rex.RexTableInputRef in project hive by apache.

the class HiveCardinalityPreservingJoinOptimization method trim.

@Override
public RelNode trim(RelBuilder relBuilder, RelNode root) {
    try {
        if (root.getInputs().size() != 1) {
            LOG.debug("Only plans where root has one input are supported. Root: {}", root);
            return root;
        }
        REL_BUILDER.set(relBuilder);
        RexBuilder rexBuilder = relBuilder.getRexBuilder();
        RelNode rootInput = root.getInput(0);
        // Build the list of RexInputRef from root input RowType
        List<RexInputRef> rootFieldList = new ArrayList<>(rootInput.getRowType().getFieldCount());
        List<String> newColumnNames = new ArrayList<>();
        for (int i = 0; i < rootInput.getRowType().getFieldList().size(); ++i) {
            RelDataTypeField relDataTypeField = rootInput.getRowType().getFieldList().get(i);
            rootFieldList.add(rexBuilder.makeInputRef(relDataTypeField.getType(), i));
            newColumnNames.add(relDataTypeField.getName());
        }
        // Bit set to gather the refs that backtrack to constant values
        BitSet constants = new BitSet();
        List<JoinedBackFields> lineages = getExpressionLineageOf(rootFieldList, rootInput, constants);
        if (lineages == null) {
            LOG.debug("Some projected field lineage can not be determined");
            return root;
        }
        // 1. Collect candidate tables for join back and map RexNodes coming from those tables to their index in the
        // rootInput row type
        // Collect all used fields from original plan
        ImmutableBitSet fieldsUsed = ImmutableBitSet.of(constants.stream().toArray());
        List<TableToJoinBack> tableToJoinBackList = new ArrayList<>(lineages.size());
        Map<Integer, RexNode> rexNodesToShuttle = new HashMap<>(rootInput.getRowType().getFieldCount());
        for (JoinedBackFields joinedBackFields : lineages) {
            Optional<ImmutableBitSet> projectedKeys = joinedBackFields.relOptHiveTable.getNonNullableKeys().stream().filter(joinedBackFields.fieldsInSourceTable::contains).findFirst();
            if (projectedKeys.isPresent() && !projectedKeys.get().equals(joinedBackFields.fieldsInSourceTable)) {
                TableToJoinBack tableToJoinBack = new TableToJoinBack(projectedKeys.get(), joinedBackFields);
                tableToJoinBackList.add(tableToJoinBack);
                fieldsUsed = fieldsUsed.union(joinedBackFields.getSource(projectedKeys.get()));
                for (TableInputRefHolder mapping : joinedBackFields.mapping) {
                    if (!fieldsUsed.get(mapping.indexInOriginalRowType)) {
                        rexNodesToShuttle.put(mapping.indexInOriginalRowType, mapping.rexNode);
                    }
                }
            } else {
                fieldsUsed = fieldsUsed.union(joinedBackFields.fieldsInOriginalRowType);
            }
        }
        if (tableToJoinBackList.isEmpty()) {
            LOG.debug("None of the tables has keys projected, unable to join back");
            return root;
        }
        // 2. Trim out non-key fields of joined back tables
        Set<RelDataTypeField> extraFields = Collections.emptySet();
        TrimResult trimResult = dispatchTrimFields(rootInput, fieldsUsed, extraFields);
        RelNode newInput = trimResult.left;
        if (newInput.getRowType().equals(rootInput.getRowType())) {
            LOG.debug("Nothing was trimmed out.");
            return root;
        }
        // 3. Join back tables to the top of original plan
        Mapping newInputMapping = trimResult.right;
        Map<RexTableInputRef, Integer> tableInputRefMapping = new HashMap<>();
        for (TableToJoinBack tableToJoinBack : tableToJoinBackList) {
            LOG.debug("Joining back table {}", tableToJoinBack.joinedBackFields.relOptHiveTable.getName());
            // 3.1. Create new TableScan of tables to join back
            RelOptHiveTable relOptTable = tableToJoinBack.joinedBackFields.relOptHiveTable;
            RelOptCluster cluster = relBuilder.getCluster();
            HiveTableScan tableScan = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), relOptTable, relOptTable.getHiveTableMD().getTableName(), null, false, false);
            // 3.2. Create Project with the required fields from this table
            RelNode projectTableAccessRel = tableScan.project(tableToJoinBack.joinedBackFields.fieldsInSourceTable, new HashSet<>(0), REL_BUILDER.get());
            // 3.3. Create mapping between the Project and TableScan
            Mapping projectMapping = Mappings.create(MappingType.INVERSE_SURJECTION, tableScan.getRowType().getFieldCount(), tableToJoinBack.joinedBackFields.fieldsInSourceTable.cardinality());
            int projectIndex = 0;
            for (int i : tableToJoinBack.joinedBackFields.fieldsInSourceTable) {
                projectMapping.set(i, projectIndex);
                ++projectIndex;
            }
            int offset = newInput.getRowType().getFieldCount();
            // 3.4. Map rexTableInputRef to the index where it can be found in the new Input row type
            for (TableInputRefHolder mapping : tableToJoinBack.joinedBackFields.mapping) {
                int indexInSourceTable = mapping.tableInputRef.getIndex();
                if (!tableToJoinBack.keys.get(indexInSourceTable)) {
                    // 3.5. if this is not a key field it is shifted by the left input field count
                    tableInputRefMapping.put(mapping.tableInputRef, offset + projectMapping.getTarget(indexInSourceTable));
                }
            }
            // 3.7. Create Join
            relBuilder.push(newInput);
            relBuilder.push(projectTableAccessRel);
            RexNode joinCondition = joinCondition(newInput, newInputMapping, tableToJoinBack, projectTableAccessRel, projectMapping, rexBuilder);
            newInput = relBuilder.join(JoinRelType.INNER, joinCondition).build();
        }
        // 4. Collect rexNodes for Project
        TableInputRefMapper mapper = new TableInputRefMapper(tableInputRefMapping, rexBuilder, newInput);
        List<RexNode> rexNodeList = new ArrayList<>(rootInput.getRowType().getFieldCount());
        for (int i = 0; i < rootInput.getRowType().getFieldCount(); i++) {
            RexNode rexNode = rexNodesToShuttle.get(i);
            if (rexNode != null) {
                rexNodeList.add(mapper.apply(rexNode));
            } else {
                int target = newInputMapping.getTarget(i);
                rexNodeList.add(rexBuilder.makeInputRef(newInput.getRowType().getFieldList().get(target).getType(), target));
            }
        }
        // 5. Create Project on top of all Join backs
        relBuilder.push(newInput);
        relBuilder.project(rexNodeList, newColumnNames);
        return root.copy(root.getTraitSet(), singletonList(relBuilder.build()));
    } finally {
        REL_BUILDER.remove();
    }
}

Example 14 with RexTableInputRef

use of org.apache.calcite.rex.RexTableInputRef in project hive by apache.

the class HiveAggregatePartitionIncrementalRewritingRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    RexBuilder rexBuilder = call.builder().getRexBuilder();
    final Aggregate aggregate = call.rel(0);
    final Union union = call.rel(1);
    final RelNode queryBranch = union.getInput(0);
    final RelNode mvBranch = union.getInput(1);
    // find Partition col indexes in mvBranch top operator row schema
    // mvBranch can be more complex than just a TS on the MV and the partition columns indexes in the top Operator's
    // row schema may differ from the one in the TS row schema. Example:
    // Project($2, $0, $1)
    // TableScan(table=materialized_view1, schema=a, b, part_col)
    RelMetadataQuery relMetadataQuery = RelMetadataQuery.instance();
    int partitionColumnCount = -1;
    List<Integer> partitionColumnIndexes = new ArrayList<>();
    for (int i = 0; i < mvBranch.getRowType().getFieldList().size(); ++i) {
        RelDataTypeField relDataTypeField = mvBranch.getRowType().getFieldList().get(i);
        RexInputRef inputRef = rexBuilder.makeInputRef(relDataTypeField.getType(), i);
        Set<RexNode> expressionLineage = relMetadataQuery.getExpressionLineage(mvBranch, inputRef);
        if (expressionLineage == null || expressionLineage.size() != 1) {
            continue;
        }
        Set<RexTableInputRef> tableInputRefs = findRexTableInputRefs(expressionLineage.iterator().next());
        if (tableInputRefs.size() != 1) {
            continue;
        }
        RexTableInputRef tableInputRef = tableInputRefs.iterator().next();
        RelOptHiveTable relOptHiveTable = (RelOptHiveTable) tableInputRef.getTableRef().getTable();
        if (!(relOptHiveTable.getHiveTableMD().isMaterializedView())) {
            LOG.warn("{} is not a materialized view, bail out.", relOptHiveTable.getQualifiedName());
            return;
        }
        partitionColumnCount = relOptHiveTable.getPartColInfoMap().size();
        if (relOptHiveTable.getPartColInfoMap().containsKey(tableInputRef.getIndex())) {
            partitionColumnIndexes.add(i);
        }
    }
    if (partitionColumnCount <= 0 || partitionColumnIndexes.size() != partitionColumnCount) {
        LOG.debug("Could not find all partition column lineages, bail out.");
        return;
    }
    List<RexNode> joinConjs = new ArrayList<>();
    for (int partColIndex : partitionColumnIndexes) {
        RexNode leftRef = rexBuilder.makeInputRef(mvBranch.getRowType().getFieldList().get(partColIndex).getType(), partColIndex);
        RexNode rightRef = rexBuilder.makeInputRef(queryBranch.getRowType().getFieldList().get(partColIndex).getType(), partColIndex + mvBranch.getRowType().getFieldCount());
        joinConjs.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_DISTINCT_FROM, leftRef, rightRef));
    }
    RexNode joinCond = RexUtil.composeConjunction(rexBuilder, joinConjs);
    RelBuilder builder = call.builder();
    RelNode newNode = builder.push(mvBranch).push(queryBranch).join(JoinRelType.SEMI, joinCond).push(queryBranch).union(union.all).aggregate(builder.groupKey(aggregate.getGroupSet()), aggregate.getAggCallList()).build();
    call.transformTo(newNode);
}

Example 15 with RexTableInputRef

use of org.apache.calcite.rex.RexTableInputRef in project hive by apache.

the class HiveCardinalityPreservingJoinOptimization method getExpressionLineageOf.

private List<JoinedBackFields> getExpressionLineageOf(List<RexInputRef> projectExpressions, RelNode projectInput, BitSet constants) {
    RelMetadataQuery relMetadataQuery = RelMetadataQuery.instance();
    Map<RexTableInputRef.RelTableRef, JoinedBackFieldsBuilder> fieldMappingBuilders = new HashMap<>();
    // use this list to keep the order of tables
    List<RexTableInputRef.RelTableRef> tablesOrdered = new ArrayList<>();
    for (RexInputRef expr : projectExpressions) {
        Set<RexNode> expressionLineage = relMetadataQuery.getExpressionLineage(projectInput, expr);
        if (expressionLineage == null || expressionLineage.size() != 1) {
            LOG.debug("Lineage of expression in node {} can not be determined: {}", projectInput, expr);
            return null;
        }
        RexNode rexNode = expressionLineage.iterator().next();
        Set<RexTableInputRef> refs = findRexTableInputRefs(rexNode);
        if (refs.isEmpty()) {
            if (!RexUtil.isConstant(rexNode)) {
                LOG.debug("Unknown expression that should be a constant: {}", rexNode);
                return null;
            }
            constants.set(expr.getIndex());
        } else {
            for (RexTableInputRef rexTableInputRef : refs) {
                RexTableInputRef.RelTableRef tableRef = rexTableInputRef.getTableRef();
                JoinedBackFieldsBuilder joinedBackFieldsBuilder = fieldMappingBuilders.computeIfAbsent(tableRef, k -> {
                    tablesOrdered.add(tableRef);
                    return new JoinedBackFieldsBuilder(tableRef);
                });
                joinedBackFieldsBuilder.add(expr, rexNode, rexTableInputRef);
            }
        }
    }
    return tablesOrdered.stream().map(relOptHiveTable -> fieldMappingBuilders.get(relOptHiveTable).build()).collect(Collectors.toList());
}