Examples with HiveRelDistribution org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelDistribution used on opensource projects

Example 1 with HiveRelDistribution

use of org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelDistribution in project hive by apache.

the class HiveAlgorithmsUtil method getJoinDistribution.

public static RelDistribution getJoinDistribution(JoinPredicateInfo joinPredInfo, MapJoinStreamingRelation streamingRelation) {
    // Compute distribution
    ImmutableList.Builder<Integer> leftKeysListBuilder = new ImmutableList.Builder<Integer>();
    ImmutableList.Builder<Integer> rightKeysListBuilder = new ImmutableList.Builder<Integer>();
    for (int i = 0; i < joinPredInfo.getEquiJoinPredicateElements().size(); i++) {
        JoinLeafPredicateInfo joinLeafPredInfo = joinPredInfo.getEquiJoinPredicateElements().get(i);
        for (int leftPos : joinLeafPredInfo.getProjsFromLeftPartOfJoinKeysInJoinSchema()) {
            leftKeysListBuilder.add(leftPos);
        }
        for (int rightPos : joinLeafPredInfo.getProjsFromRightPartOfJoinKeysInJoinSchema()) {
            rightKeysListBuilder.add(rightPos);
        }
    }
    RelDistribution distribution = null;
    // Keep buckets from the streaming relation
    if (streamingRelation == MapJoinStreamingRelation.LEFT_RELATION) {
        distribution = new HiveRelDistribution(RelDistribution.Type.HASH_DISTRIBUTED, leftKeysListBuilder.build());
    } else if (streamingRelation == MapJoinStreamingRelation.RIGHT_RELATION) {
        distribution = new HiveRelDistribution(RelDistribution.Type.HASH_DISTRIBUTED, rightKeysListBuilder.build());
    }
    return distribution;
}

Example 2 with HiveRelDistribution

use of org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelDistribution in project hive by apache.

the class HiveProjectSortExchangeTransposeRule method onMatch.

// ~ Methods ----------------------------------------------------------------
// implement RelOptRule
public void onMatch(RelOptRuleCall call) {
    final HiveProject project = call.rel(0);
    final HiveSortExchange sortExchange = call.rel(1);
    final RelOptCluster cluster = project.getCluster();
    List<RelFieldCollation> fieldCollations = getNewRelFieldCollations(project, sortExchange.getCollation(), cluster);
    if (fieldCollations == null) {
        return;
    }
    RelCollation newCollation = RelCollationTraitDef.INSTANCE.canonize(RelCollationImpl.of(fieldCollations));
    List<Integer> newDistributionKeys = getNewRelDistributionKeys(project, sortExchange.getDistribution());
    RelDistribution newDistribution = RelDistributionTraitDef.INSTANCE.canonize(new HiveRelDistribution(sortExchange.getDistribution().getType(), newDistributionKeys));
    RelTraitSet newTraitSet = TraitsUtil.getDefaultTraitSet(sortExchange.getCluster()).replace(newCollation).replace(newDistribution);
    // New operators
    final RelNode newProject = project.copy(sortExchange.getInput().getTraitSet(), ImmutableList.of(sortExchange.getInput()));
    final SortExchange newSort = sortExchange.copy(newTraitSet, newProject, newDistribution, newCollation);
    call.transformTo(newSort);
}

Example 3 with HiveRelDistribution

use of org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelDistribution in project hive by apache.

the class HiveInsertExchange4JoinRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    JoinPredicateInfo joinPredInfo;
    if (call.rel(0) instanceof HiveMultiJoin) {
        HiveMultiJoin multiJoin = call.rel(0);
        joinPredInfo = multiJoin.getJoinPredicateInfo();
    } else if (call.rel(0) instanceof HiveJoin) {
        HiveJoin hiveJoin = call.rel(0);
        joinPredInfo = hiveJoin.getJoinPredicateInfo();
    } else if (call.rel(0) instanceof Join) {
        Join join = call.rel(0);
        try {
            joinPredInfo = HiveCalciteUtil.JoinPredicateInfo.constructJoinPredicateInfo(join);
        } catch (CalciteSemanticException e) {
            throw new RuntimeException(e);
        }
    } else {
        return;
    }
    for (RelNode child : call.rel(0).getInputs()) {
        if (((HepRelVertex) child).getCurrentRel() instanceof Exchange) {
            return;
        }
    }
    // Get key columns from inputs. Those are the columns on which we will distribute on.
    // It is also the columns we will sort on.
    List<RelNode> newInputs = new ArrayList<RelNode>();
    for (int i = 0; i < call.rel(0).getInputs().size(); i++) {
        List<Integer> joinKeyPositions = new ArrayList<Integer>();
        ImmutableList.Builder<RexNode> joinExprsBuilder = new ImmutableList.Builder<RexNode>();
        Set<String> keySet = Sets.newHashSet();
        ImmutableList.Builder<RelFieldCollation> collationListBuilder = new ImmutableList.Builder<RelFieldCollation>();
        for (int j = 0; j < joinPredInfo.getEquiJoinPredicateElements().size(); j++) {
            JoinLeafPredicateInfo joinLeafPredInfo = joinPredInfo.getEquiJoinPredicateElements().get(j);
            for (RexNode joinExprNode : joinLeafPredInfo.getJoinExprs(i)) {
                if (keySet.add(joinExprNode.toString())) {
                    joinExprsBuilder.add(joinExprNode);
                }
            }
            for (int pos : joinLeafPredInfo.getProjsJoinKeysInChildSchema(i)) {
                if (!joinKeyPositions.contains(pos)) {
                    joinKeyPositions.add(pos);
                    collationListBuilder.add(new RelFieldCollation(pos));
                }
            }
        }
        HiveSortExchange exchange = HiveSortExchange.create(call.rel(0).getInput(i), new HiveRelDistribution(RelDistribution.Type.HASH_DISTRIBUTED, joinKeyPositions), new HiveRelCollation(collationListBuilder.build()), joinExprsBuilder.build());
        newInputs.add(exchange);
    }
    RelNode newOp;
    if (call.rel(0) instanceof HiveMultiJoin) {
        HiveMultiJoin multiJoin = call.rel(0);
        newOp = multiJoin.copy(multiJoin.getTraitSet(), newInputs);
    } else if (call.rel(0) instanceof Join) {
        Join join = call.rel(0);
        newOp = join.copy(join.getTraitSet(), join.getCondition(), newInputs.get(0), newInputs.get(1), join.getJoinType(), join.isSemiJoinDone());
    } else {
        return;
    }
    call.getPlanner().onCopy(call.rel(0), newOp);
    call.transformTo(newOp);
}