Examples with DistributionField org.apache.drill.exec.planner.physical.DrillDistributionTrait.DistributionField used on opensource projects

Example 1 with DistributionField

use of org.apache.drill.exec.planner.physical.DrillDistributionTrait.DistributionField in project drill by apache.

the class InsertLocalExchangeVisitor method visitExchange.

@Override
public Prel visitExchange(ExchangePrel prel, Void value) throws RuntimeException {
    Prel child = ((Prel) prel.getInput()).accept(this, null);
    //   If DeMuxExchange is enabled, insert a UnorderedDeMuxExchangePrel after HashToRandomExchangePrel.
    if (!(prel instanceof HashToRandomExchangePrel)) {
        return (Prel) prel.copy(prel.getTraitSet(), Collections.singletonList(((RelNode) child)));
    }
    Prel newPrel = child;
    final HashToRandomExchangePrel hashPrel = (HashToRandomExchangePrel) prel;
    final List<String> childFields = child.getRowType().getFieldNames();
    List<RexNode> removeUpdatedExpr = null;
    if (isMuxEnabled) {
        // Insert Project Operator with new column that will be a hash for HashToRandomExchange fields
        final List<DistributionField> distFields = hashPrel.getFields();
        final List<String> outputFieldNames = Lists.newArrayList(childFields);
        final RexBuilder rexBuilder = prel.getCluster().getRexBuilder();
        final List<RelDataTypeField> childRowTypeFields = child.getRowType().getFieldList();
        final HashExpressionCreatorHelper<RexNode> hashHelper = new RexNodeBasedHashExpressionCreatorHelper(rexBuilder);
        final List<RexNode> distFieldRefs = Lists.newArrayListWithExpectedSize(distFields.size());
        for (int i = 0; i < distFields.size(); i++) {
            final int fieldId = distFields.get(i).getFieldId();
            distFieldRefs.add(rexBuilder.makeInputRef(childRowTypeFields.get(fieldId).getType(), fieldId));
        }
        final List<RexNode> updatedExpr = Lists.newArrayListWithExpectedSize(childRowTypeFields.size());
        removeUpdatedExpr = Lists.newArrayListWithExpectedSize(childRowTypeFields.size());
        for (RelDataTypeField field : childRowTypeFields) {
            RexNode rex = rexBuilder.makeInputRef(field.getType(), field.getIndex());
            updatedExpr.add(rex);
            removeUpdatedExpr.add(rex);
        }
        outputFieldNames.add(HashPrelUtil.HASH_EXPR_NAME);
        // distribution seed
        final RexNode distSeed = rexBuilder.makeBigintLiteral(BigDecimal.valueOf(HashPrelUtil.DIST_SEED));
        updatedExpr.add(HashPrelUtil.createHashBasedPartitionExpression(distFieldRefs, distSeed, hashHelper));
        RelDataType rowType = RexUtil.createStructType(prel.getCluster().getTypeFactory(), updatedExpr, outputFieldNames);
        ProjectPrel addColumnprojectPrel = new ProjectPrel(child.getCluster(), child.getTraitSet(), child, updatedExpr, rowType);
        newPrel = new UnorderedMuxExchangePrel(addColumnprojectPrel.getCluster(), addColumnprojectPrel.getTraitSet(), addColumnprojectPrel);
    }
    newPrel = new HashToRandomExchangePrel(prel.getCluster(), prel.getTraitSet(), newPrel, ((HashToRandomExchangePrel) prel).getFields());
    if (isDeMuxEnabled) {
        HashToRandomExchangePrel hashExchangePrel = (HashToRandomExchangePrel) newPrel;
        // Insert a DeMuxExchange to narrow down the number of receivers
        newPrel = new UnorderedDeMuxExchangePrel(prel.getCluster(), prel.getTraitSet(), hashExchangePrel, hashExchangePrel.getFields());
    }
    if (isMuxEnabled) {
        // remove earlier inserted Project Operator - since it creates issues down the road in HashJoin
        RelDataType removeRowType = RexUtil.createStructType(newPrel.getCluster().getTypeFactory(), removeUpdatedExpr, childFields);
        ProjectPrel removeColumnProjectPrel = new ProjectPrel(newPrel.getCluster(), newPrel.getTraitSet(), newPrel, removeUpdatedExpr, removeRowType);
        return removeColumnProjectPrel;
    }
    return newPrel;
}

Example 2 with DistributionField

use of org.apache.drill.exec.planner.physical.DrillDistributionTrait.DistributionField in project drill by apache.

the class AggPruleBase method getDistributionField.

protected List<DistributionField> getDistributionField(DrillAggregateRel rel, boolean allFields) {
    List<DistributionField> groupByFields = Lists.newArrayList();
    for (int group : BitSets.toIter(rel.getGroupSet())) {
        DistributionField field = new DistributionField(group);
        groupByFields.add(field);
        if (!allFields && groupByFields.size() == 1) {
            // with highest NDV.
            break;
        }
    }
    return groupByFields;
}

Example 3 with DistributionField

use of org.apache.drill.exec.planner.physical.DrillDistributionTrait.DistributionField in project drill by apache.

the class UnionAllPrule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final DrillUnionRel union = (DrillUnionRel) call.rel(0);
    final List<RelNode> inputs = union.getInputs();
    List<RelNode> convertedInputList = Lists.newArrayList();
    PlannerSettings settings = PrelUtil.getPlannerSettings(call.getPlanner());
    boolean allHashDistributed = true;
    for (int i = 0; i < inputs.size(); i++) {
        RelNode child = inputs.get(i);
        List<DistributionField> childDistFields = Lists.newArrayList();
        RelNode convertedChild;
        for (RelDataTypeField f : child.getRowType().getFieldList()) {
            childDistFields.add(new DistributionField(f.getIndex()));
        }
        if (settings.isUnionAllDistributeEnabled()) {
            /*
         * Strictly speaking, union-all does not need re-distribution of data; but in Drill's execution
         * model, the data distribution and parallelism operators are the same. Here, we insert a
         * hash distribution operator to allow parallelism to be determined independently for the parent
         * and children. (See DRILL-4833).
         * Note that a round robin distribution would have sufficed but we don't have one.
         */
            DrillDistributionTrait hashChild = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(childDistFields));
            RelTraitSet traitsChild = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(hashChild);
            convertedChild = convert(child, PrelUtil.fixTraits(call, traitsChild));
        } else {
            RelTraitSet traitsChild = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL);
            convertedChild = convert(child, PrelUtil.fixTraits(call, traitsChild));
            allHashDistributed = false;
        }
        convertedInputList.add(convertedChild);
    }
    try {
        RelTraitSet traits;
        if (allHashDistributed) {
            // since all children of union-all are hash distributed, propagate the traits of the left child
            traits = convertedInputList.get(0).getTraitSet();
        } else {
            // output distribution trait is set to ANY since union-all inputs may be distributed in different ways
            // and unlike a join there are no join keys that allow determining how the output would be distributed.
            // Note that a downstream operator may impose a required distribution which would be satisfied by
            // inserting an Exchange after the Union-All.
            traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.ANY);
        }
        Preconditions.checkArgument(convertedInputList.size() >= 2, "Union list must be at least two items.");
        RelNode left = convertedInputList.get(0);
        for (int i = 1; i < convertedInputList.size(); i++) {
            left = new UnionAllPrel(union.getCluster(), traits, ImmutableList.of(left, convertedInputList.get(i)), false);
        }
        call.transformTo(left);
    } catch (InvalidRelException e) {
        tracer.warning(e.toString());
    }
}

Example 4 with DistributionField

use of org.apache.drill.exec.planner.physical.DrillDistributionTrait.DistributionField in project drill by apache.

the class WindowPrule method getDistributionFieldsFromCollation.

private List<DistributionField> getDistributionFieldsFromCollation(Window.Group window) {
    List<DistributionField> distFields = Lists.newArrayList();
    for (RelFieldCollation relField : window.collation().getFieldCollations()) {
        DistributionField field = new DistributionField(relField.getFieldIndex());
        distFields.add(field);
    }
    return distFields;
}

Example 5 with DistributionField

use of org.apache.drill.exec.planner.physical.DrillDistributionTrait.DistributionField in project drill by apache.

the class SortPrule method getDistributionField.

private List<DistributionField> getDistributionField(DrillSortRel rel) {
    List<DistributionField> distFields = Lists.newArrayList();
    for (RelFieldCollation relField : rel.getCollation().getFieldCollations()) {
        DistributionField field = new DistributionField(relField.getFieldIndex());
        distFields.add(field);
    }
    return distFields;
}