Examples with JoinInfo org.apache.calcite.rel.core.JoinInfo used on opensource projects

Example 6 with JoinInfo

use of org.apache.calcite.rel.core.JoinInfo in project calcite by apache.

the class JoinAddRedundantSemiJoinRule method onMatch.

// ~ Methods ----------------------------------------------------------------
public void onMatch(RelOptRuleCall call) {
    Join origJoinRel = call.rel(0);
    if (origJoinRel.isSemiJoinDone()) {
        return;
    }
    // can't process outer joins using semijoins
    if (origJoinRel.getJoinType() != JoinRelType.INNER) {
        return;
    }
    // determine if we have a valid join condition
    final JoinInfo joinInfo = origJoinRel.analyzeCondition();
    if (joinInfo.leftKeys.size() == 0) {
        return;
    }
    RelNode semiJoin = SemiJoin.create(origJoinRel.getLeft(), origJoinRel.getRight(), origJoinRel.getCondition(), joinInfo.leftKeys, joinInfo.rightKeys);
    RelNode newJoinRel = origJoinRel.copy(origJoinRel.getTraitSet(), origJoinRel.getCondition(), semiJoin, origJoinRel.getRight(), JoinRelType.INNER, true);
    call.transformTo(newJoinRel);
}

Example 7 with JoinInfo

use of org.apache.calcite.rel.core.JoinInfo in project calcite by apache.

the class LoptSemiJoinOptimizer method findSemiJoinIndexByCost.

/**
 * Given a list of possible filters on a fact table, determine if there is
 * an index that can be used, provided all the fact table keys originate
 * from the same underlying table.
 *
 * @param multiJoin join factors being optimized
 * @param joinFilters filters to be used on the fact table
 * @param factIdx index in join factors corresponding to the fact table
 * @param dimIdx index in join factors corresponding to the dimension table
 *
 * @return SemiJoin containing information regarding the semijoin that
 * can be used to filter the fact table
 */
private SemiJoin findSemiJoinIndexByCost(LoptMultiJoin multiJoin, List<RexNode> joinFilters, int factIdx, int dimIdx) {
    // create a SemiJoin with the semi-join condition and keys
    RexNode semiJoinCondition = RexUtil.composeConjunction(rexBuilder, joinFilters, true);
    int leftAdjustment = 0;
    for (int i = 0; i < factIdx; i++) {
        leftAdjustment -= multiJoin.getNumFieldsInJoinFactor(i);
    }
    semiJoinCondition = adjustSemiJoinCondition(multiJoin, leftAdjustment, semiJoinCondition, factIdx, dimIdx);
    RelNode factRel = multiJoin.getJoinFactor(factIdx);
    RelNode dimRel = multiJoin.getJoinFactor(dimIdx);
    final JoinInfo joinInfo = JoinInfo.of(factRel, dimRel, semiJoinCondition);
    assert joinInfo.leftKeys.size() > 0;
    // mutable copies
    final List<Integer> leftKeys = Lists.newArrayList(joinInfo.leftKeys);
    final List<Integer> rightKeys = Lists.newArrayList(joinInfo.rightKeys);
    // make sure all the fact table keys originate from the same table
    // and are simple column references
    final List<Integer> actualLeftKeys = new ArrayList<>();
    LcsTable factTable = validateKeys(factRel, leftKeys, rightKeys, actualLeftKeys);
    if (factTable == null) {
        return null;
    }
    // find the best index
    final List<Integer> bestKeyOrder = new ArrayList<>();
    LcsTableScan tmpFactRel = (LcsTableScan) factTable.toRel(RelOptUtil.getContext(factRel.getCluster()));
    LcsIndexOptimizer indexOptimizer = new LcsIndexOptimizer(tmpFactRel);
    FemLocalIndex bestIndex = indexOptimizer.findSemiJoinIndexByCost(dimRel, actualLeftKeys, rightKeys, bestKeyOrder);
    if (bestIndex == null) {
        return null;
    }
    // if necessary, truncate the keys to reflect the ones that match
    // the index and remove the corresponding, unnecessary filters from
    // the condition; note that we don't save the actual keys here because
    // later when the semijoin is pushed past other RelNodes, the keys will
    // be converted
    final List<Integer> truncatedLeftKeys;
    final List<Integer> truncatedRightKeys;
    if (actualLeftKeys.size() == bestKeyOrder.size()) {
        truncatedLeftKeys = leftKeys;
        truncatedRightKeys = rightKeys;
    } else {
        truncatedLeftKeys = new ArrayList<>();
        truncatedRightKeys = new ArrayList<>();
        for (int key : bestKeyOrder) {
            truncatedLeftKeys.add(leftKeys.get(key));
            truncatedRightKeys.add(rightKeys.get(key));
        }
        semiJoinCondition = removeExtraFilters(truncatedLeftKeys, multiJoin.getNumFieldsInJoinFactor(factIdx), semiJoinCondition);
    }
    return SemiJoin.create(factRel, dimRel, semiJoinCondition, ImmutableIntList.copyOf(truncatedLeftKeys), ImmutableIntList.copyOf(truncatedRightKeys));
}

Example 8 with JoinInfo

use of org.apache.calcite.rel.core.JoinInfo in project calcite by apache.

the class SqlToRelConverter method translateIn.

private RexNode translateIn(RelOptUtil.Logic logic, RelNode root, final RexNode rex) {
    switch(logic) {
        case TRUE:
            return rexBuilder.makeLiteral(true);
        case TRUE_FALSE:
        case UNKNOWN_AS_FALSE:
            assert rex instanceof RexRangeRef;
            final int fieldCount = rex.getType().getFieldCount();
            RexNode rexNode = rexBuilder.makeFieldAccess(rex, fieldCount - 1);
            rexNode = rexBuilder.makeCall(SqlStdOperatorTable.IS_TRUE, rexNode);
            // Then append the IS NOT NULL(leftKeysForIn).
            // 
            // RexRangeRef contains the following fields:
            // leftKeysForIn,
            // rightKeysForIn (the original sub-query select list),
            // nullIndicator
            // 
            // The first two lists contain the same number of fields.
            final int k = (fieldCount - 1) / 2;
            for (int i = 0; i < k; i++) {
                rexNode = rexBuilder.makeCall(SqlStdOperatorTable.AND, rexNode, rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, rexBuilder.makeFieldAccess(rex, i)));
            }
            return rexNode;
        case TRUE_FALSE_UNKNOWN:
        case UNKNOWN_AS_TRUE:
            // select e.deptno,
            // case
            // when ct.c = 0 then false
            // when dt.i is not null then true
            // when e.deptno is null then null
            // when ct.ck < ct.c then null
            // else false
            // end
            // from e
            // cross join (select count(*) as c, count(deptno) as ck from v) as ct
            // left join (select distinct deptno, true as i from v) as dt
            // on e.deptno = dt.deptno
            final Join join = (Join) root;
            final Project left = (Project) join.getLeft();
            final RelNode leftLeft = ((Join) left.getInput()).getLeft();
            final int leftLeftCount = leftLeft.getRowType().getFieldCount();
            final RelDataType longType = typeFactory.createSqlType(SqlTypeName.BIGINT);
            final RexNode cRef = rexBuilder.makeInputRef(root, leftLeftCount);
            final RexNode ckRef = rexBuilder.makeInputRef(root, leftLeftCount + 1);
            final RexNode iRef = rexBuilder.makeInputRef(root, root.getRowType().getFieldCount() - 1);
            final RexLiteral zero = rexBuilder.makeExactLiteral(BigDecimal.ZERO, longType);
            final RexLiteral trueLiteral = rexBuilder.makeLiteral(true);
            final RexLiteral falseLiteral = rexBuilder.makeLiteral(false);
            final RexNode unknownLiteral = rexBuilder.makeNullLiteral(trueLiteral.getType());
            final ImmutableList.Builder<RexNode> args = ImmutableList.builder();
            args.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, cRef, zero), falseLiteral, rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, iRef), trueLiteral);
            final JoinInfo joinInfo = join.analyzeCondition();
            for (int leftKey : joinInfo.leftKeys) {
                final RexNode kRef = rexBuilder.makeInputRef(root, leftKey);
                args.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, kRef), unknownLiteral);
            }
            args.add(rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, ckRef, cRef), unknownLiteral, falseLiteral);
            return rexBuilder.makeCall(SqlStdOperatorTable.CASE, args.build());
        default:
            throw new AssertionError(logic);
    }
}

Example 9 with JoinInfo

use of org.apache.calcite.rel.core.JoinInfo in project calcite by apache.

the class SemiJoinRule method perform.

protected void perform(RelOptRuleCall call, Project project, Join join, RelNode left, Aggregate aggregate) {
    final RelOptCluster cluster = join.getCluster();
    final RexBuilder rexBuilder = cluster.getRexBuilder();
    if (project != null) {
        final ImmutableBitSet bits = RelOptUtil.InputFinder.bits(project.getProjects(), null);
        final ImmutableBitSet rightBits = ImmutableBitSet.range(left.getRowType().getFieldCount(), join.getRowType().getFieldCount());
        if (bits.intersects(rightBits)) {
            return;
        }
    }
    final JoinInfo joinInfo = join.analyzeCondition();
    if (!joinInfo.rightSet().equals(ImmutableBitSet.range(aggregate.getGroupCount()))) {
        // By the way, neither a super-set nor a sub-set would work.
        return;
    }
    if (!joinInfo.isEqui()) {
        return;
    }
    final RelBuilder relBuilder = call.builder();
    relBuilder.push(left);
    switch(join.getJoinType()) {
        case INNER:
            final List<Integer> newRightKeyBuilder = Lists.newArrayList();
            final List<Integer> aggregateKeys = aggregate.getGroupSet().asList();
            for (int key : joinInfo.rightKeys) {
                newRightKeyBuilder.add(aggregateKeys.get(key));
            }
            final ImmutableIntList newRightKeys = ImmutableIntList.copyOf(newRightKeyBuilder);
            relBuilder.push(aggregate.getInput());
            final RexNode newCondition = RelOptUtil.createEquiJoinCondition(relBuilder.peek(2, 0), joinInfo.leftKeys, relBuilder.peek(2, 1), newRightKeys, rexBuilder);
            relBuilder.semiJoin(newCondition);
            break;
        case LEFT:
            // we can eliminate the semi-join.
            break;
        default:
            throw new AssertionError(join.getJoinType());
    }
    if (project != null) {
        relBuilder.project(project.getProjects(), project.getRowType().getFieldNames());
    }
    call.transformTo(relBuilder.build());
}

Example 10 with JoinInfo

use of org.apache.calcite.rel.core.JoinInfo in project drill by apache.

the class RuntimeFilterVisitor method generateRuntimeFilter.

/**
 * Generate a possible RuntimeFilter of a HashJoinPrel, left some BF parameters of the generated RuntimeFilter
 * to be set later.
 *
 * @param hashJoinPrel
 * @return null or a partial information RuntimeFilterDef
 */
private RuntimeFilterDef generateRuntimeFilter(HashJoinPrel hashJoinPrel) {
    JoinRelType joinRelType = hashJoinPrel.getJoinType();
    JoinInfo joinInfo = hashJoinPrel.analyzeCondition();
    boolean allowJoin = (joinInfo.isEqui()) && (joinRelType == JoinRelType.INNER || joinRelType == JoinRelType.RIGHT);
    if (!allowJoin) {
        return null;
    }
    // TODO check whether to enable RuntimeFilter according to the NDV percent
    /**
     *     double threshold = 0.5;
     *     double percent = leftNDV / rightDNV;
     *     if (percent > threshold ) {
     *     return null;
     *     }
     */
    List<BloomFilterDef> bloomFilterDefs = new ArrayList<>();
    // find the possible left scan node of the left join key
    ScanPrel probeSideScanPrel = null;
    RelNode left = hashJoinPrel.getLeft();
    RelNode right = hashJoinPrel.getRight();
    ExchangePrel exchangePrel = findRightExchangePrel(right);
    if (exchangePrel == null) {
        // can only be BroadcastExchangePrel or HashToRandomExchangePrel
        return null;
    }
    List<String> leftFields = left.getRowType().getFieldNames();
    List<String> rightFields = right.getRowType().getFieldNames();
    List<Integer> leftKeys = hashJoinPrel.getLeftKeys();
    List<Integer> rightKeys = hashJoinPrel.getRightKeys();
    RelMetadataQuery metadataQuery = left.getCluster().getMetadataQuery();
    int i = 0;
    for (Integer leftKey : leftKeys) {
        String leftFieldName = leftFields.get(leftKey);
        Integer rightKey = rightKeys.get(i++);
        String rightFieldName = rightFields.get(rightKey);
        // This also avoids the left field of the join condition with a function call.
        ScanPrel scanPrel = findLeftScanPrel(leftFieldName, left);
        if (scanPrel != null) {
            boolean encounteredBlockNode = containBlockNode((Prel) left, scanPrel);
            if (encounteredBlockNode) {
                continue;
            }
            // Collect NDV from the Metadata
            RelDataType scanRowType = scanPrel.getRowType();
            RelDataTypeField field = scanRowType.getField(leftFieldName, true, true);
            int index = field.getIndex();
            Double ndv = metadataQuery.getDistinctRowCount(scanPrel, ImmutableBitSet.of(index), null);
            if (ndv == null) {
                // If NDV is not supplied, we use the row count to estimate the ndv.
                ndv = left.estimateRowCount(metadataQuery) * 0.1;
            }
            int bloomFilterSizeInBytes = BloomFilter.optimalNumOfBytes(ndv.longValue(), fpp);
            bloomFilterSizeInBytes = bloomFilterSizeInBytes > bloomFilterMaxSizeInBytesDef ? bloomFilterMaxSizeInBytesDef : bloomFilterSizeInBytes;
            // left the local parameter to be set later.
            BloomFilterDef bloomFilterDef = new BloomFilterDef(bloomFilterSizeInBytes, false, leftFieldName, rightFieldName);
            bloomFilterDef.setLeftNDV(ndv);
            bloomFilterDefs.add(bloomFilterDef);
            toAddRuntimeFilter.add(scanPrel);
            probeSideScanPrel = scanPrel;
        }
    }
    if (bloomFilterDefs.size() > 0) {
        // left sendToForeman parameter to be set later.
        RuntimeFilterDef runtimeFilterDef = new RuntimeFilterDef(true, false, bloomFilterDefs, false, -1);
        probeSideScan2hj.put(probeSideScanPrel, hashJoinPrel);
        return runtimeFilterDef;
    }
    return null;
}