Examples with ExchangePrel org.apache.drill.exec.planner.physical.ExchangePrel used on opensource projects

Example 1 with ExchangePrel

use of org.apache.drill.exec.planner.physical.ExchangePrel in project drill by apache.

the class InsertLocalExchangeVisitor method visitExchange.

@Override
public Prel visitExchange(ExchangePrel prel, Void value) throws RuntimeException {
    Prel child = ((Prel) prel.getInput()).accept(this, null);
    //   If DeMuxExchange is enabled, insert a UnorderedDeMuxExchangePrel after HashToRandomExchangePrel.
    if (!(prel instanceof HashToRandomExchangePrel)) {
        return (Prel) prel.copy(prel.getTraitSet(), Collections.singletonList(((RelNode) child)));
    }
    Prel newPrel = child;
    final HashToRandomExchangePrel hashPrel = (HashToRandomExchangePrel) prel;
    final List<String> childFields = child.getRowType().getFieldNames();
    List<RexNode> removeUpdatedExpr = null;
    if (isMuxEnabled) {
        // Insert Project Operator with new column that will be a hash for HashToRandomExchange fields
        final List<DistributionField> distFields = hashPrel.getFields();
        final List<String> outputFieldNames = Lists.newArrayList(childFields);
        final RexBuilder rexBuilder = prel.getCluster().getRexBuilder();
        final List<RelDataTypeField> childRowTypeFields = child.getRowType().getFieldList();
        final HashExpressionCreatorHelper<RexNode> hashHelper = new RexNodeBasedHashExpressionCreatorHelper(rexBuilder);
        final List<RexNode> distFieldRefs = Lists.newArrayListWithExpectedSize(distFields.size());
        for (int i = 0; i < distFields.size(); i++) {
            final int fieldId = distFields.get(i).getFieldId();
            distFieldRefs.add(rexBuilder.makeInputRef(childRowTypeFields.get(fieldId).getType(), fieldId));
        }
        final List<RexNode> updatedExpr = Lists.newArrayListWithExpectedSize(childRowTypeFields.size());
        removeUpdatedExpr = Lists.newArrayListWithExpectedSize(childRowTypeFields.size());
        for (RelDataTypeField field : childRowTypeFields) {
            RexNode rex = rexBuilder.makeInputRef(field.getType(), field.getIndex());
            updatedExpr.add(rex);
            removeUpdatedExpr.add(rex);
        }
        outputFieldNames.add(HashPrelUtil.HASH_EXPR_NAME);
        // distribution seed
        final RexNode distSeed = rexBuilder.makeBigintLiteral(BigDecimal.valueOf(HashPrelUtil.DIST_SEED));
        updatedExpr.add(HashPrelUtil.createHashBasedPartitionExpression(distFieldRefs, distSeed, hashHelper));
        RelDataType rowType = RexUtil.createStructType(prel.getCluster().getTypeFactory(), updatedExpr, outputFieldNames);
        ProjectPrel addColumnprojectPrel = new ProjectPrel(child.getCluster(), child.getTraitSet(), child, updatedExpr, rowType);
        newPrel = new UnorderedMuxExchangePrel(addColumnprojectPrel.getCluster(), addColumnprojectPrel.getTraitSet(), addColumnprojectPrel);
    }
    newPrel = new HashToRandomExchangePrel(prel.getCluster(), prel.getTraitSet(), newPrel, ((HashToRandomExchangePrel) prel).getFields());
    if (isDeMuxEnabled) {
        HashToRandomExchangePrel hashExchangePrel = (HashToRandomExchangePrel) newPrel;
        // Insert a DeMuxExchange to narrow down the number of receivers
        newPrel = new UnorderedDeMuxExchangePrel(prel.getCluster(), prel.getTraitSet(), hashExchangePrel, hashExchangePrel.getFields());
    }
    if (isMuxEnabled) {
        // remove earlier inserted Project Operator - since it creates issues down the road in HashJoin
        RelDataType removeRowType = RexUtil.createStructType(newPrel.getCluster().getTypeFactory(), removeUpdatedExpr, childFields);
        ProjectPrel removeColumnProjectPrel = new ProjectPrel(newPrel.getCluster(), newPrel.getTraitSet(), newPrel, removeUpdatedExpr, removeRowType);
        return removeColumnProjectPrel;
    }
    return newPrel;
}

Example 2 with ExchangePrel

use of org.apache.drill.exec.planner.physical.ExchangePrel in project drill by apache.

the class ExcessiveExchangeIdentifier method visitExchange.

@Override
public Prel visitExchange(ExchangePrel prel, MajorFragmentStat parent) throws RuntimeException {
    parent.add(prel);
    MajorFragmentStat newFrag = new MajorFragmentStat();
    newFrag.setRightSideOfLateral(parent.isRightSideOfLateral());
    if (prel instanceof SingleMergeExchangePrel) {
        newFrag.isSimpleRel = true;
    }
    Prel newChild = ((Prel) prel.getInput()).accept(this, newFrag);
    if (parent.isSimpleRel && prel instanceof HashToMergeExchangePrel) {
        return newChild;
    }
    if (canRemoveExchange(parent, newFrag)) {
        return newChild;
    } else if (newChild != prel.getInput()) {
        return (Prel) prel.copy(prel.getTraitSet(), Collections.singletonList(newChild));
    } else {
        return prel;
    }
}

Example 3 with ExchangePrel

use of org.apache.drill.exec.planner.physical.ExchangePrel in project drill by apache.

the class RuntimeFilterVisitor method generateRuntimeFilter.

/**
 * Generate a possible RuntimeFilter of a HashJoinPrel, left some BF parameters of the generated RuntimeFilter
 * to be set later.
 *
 * @param hashJoinPrel
 * @return null or a partial information RuntimeFilterDef
 */
private RuntimeFilterDef generateRuntimeFilter(HashJoinPrel hashJoinPrel) {
    JoinRelType joinRelType = hashJoinPrel.getJoinType();
    JoinInfo joinInfo = hashJoinPrel.analyzeCondition();
    boolean allowJoin = (joinInfo.isEqui()) && (joinRelType == JoinRelType.INNER || joinRelType == JoinRelType.RIGHT);
    if (!allowJoin) {
        return null;
    }
    // TODO check whether to enable RuntimeFilter according to the NDV percent
    /**
     *     double threshold = 0.5;
     *     double percent = leftNDV / rightDNV;
     *     if (percent > threshold ) {
     *     return null;
     *     }
     */
    List<BloomFilterDef> bloomFilterDefs = new ArrayList<>();
    // find the possible left scan node of the left join key
    ScanPrel probeSideScanPrel = null;
    RelNode left = hashJoinPrel.getLeft();
    RelNode right = hashJoinPrel.getRight();
    ExchangePrel exchangePrel = findRightExchangePrel(right);
    if (exchangePrel == null) {
        // can only be BroadcastExchangePrel or HashToRandomExchangePrel
        return null;
    }
    List<String> leftFields = left.getRowType().getFieldNames();
    List<String> rightFields = right.getRowType().getFieldNames();
    List<Integer> leftKeys = hashJoinPrel.getLeftKeys();
    List<Integer> rightKeys = hashJoinPrel.getRightKeys();
    RelMetadataQuery metadataQuery = left.getCluster().getMetadataQuery();
    int i = 0;
    for (Integer leftKey : leftKeys) {
        String leftFieldName = leftFields.get(leftKey);
        Integer rightKey = rightKeys.get(i++);
        String rightFieldName = rightFields.get(rightKey);
        // This also avoids the left field of the join condition with a function call.
        ScanPrel scanPrel = findLeftScanPrel(leftFieldName, left);
        if (scanPrel != null) {
            boolean encounteredBlockNode = containBlockNode((Prel) left, scanPrel);
            if (encounteredBlockNode) {
                continue;
            }
            // Collect NDV from the Metadata
            RelDataType scanRowType = scanPrel.getRowType();
            RelDataTypeField field = scanRowType.getField(leftFieldName, true, true);
            int index = field.getIndex();
            Double ndv = metadataQuery.getDistinctRowCount(scanPrel, ImmutableBitSet.of(index), null);
            if (ndv == null) {
                // If NDV is not supplied, we use the row count to estimate the ndv.
                ndv = left.estimateRowCount(metadataQuery) * 0.1;
            }
            int bloomFilterSizeInBytes = BloomFilter.optimalNumOfBytes(ndv.longValue(), fpp);
            bloomFilterSizeInBytes = bloomFilterSizeInBytes > bloomFilterMaxSizeInBytesDef ? bloomFilterMaxSizeInBytesDef : bloomFilterSizeInBytes;
            // left the local parameter to be set later.
            BloomFilterDef bloomFilterDef = new BloomFilterDef(bloomFilterSizeInBytes, false, leftFieldName, rightFieldName);
            bloomFilterDef.setLeftNDV(ndv);
            bloomFilterDefs.add(bloomFilterDef);
            toAddRuntimeFilter.add(scanPrel);
            probeSideScanPrel = scanPrel;
        }
    }
    if (bloomFilterDefs.size() > 0) {
        // left sendToForeman parameter to be set later.
        RuntimeFilterDef runtimeFilterDef = new RuntimeFilterDef(true, false, bloomFilterDefs, false, -1);
        probeSideScan2hj.put(probeSideScanPrel, hashJoinPrel);
        return runtimeFilterDef;
    }
    return null;
}

Example 4 with ExchangePrel

use of org.apache.drill.exec.planner.physical.ExchangePrel in project drill by axbaretto.

the class PrelSequencer method visitExchange.

@Override
public Void visitExchange(ExchangePrel prel, Frag value) throws RuntimeException {
    Frag newFrag = new Frag(prel);
    frags.add(newFrag);
    value.children.add(newFrag);
    for (Prel child : prel) {
        child.accept(this, newFrag);
    }
    return null;
}

Example 5 with ExchangePrel

use of org.apache.drill.exec.planner.physical.ExchangePrel in project drill by axbaretto.

the class PrelSequencer method go.

public Map<Prel, OpId> go(Prel root) {
    // get fragments.
    Frag rootFrag = new Frag(root);
    frags.add(rootFrag);
    root.accept(this, rootFrag);
    // do depth first traversal of fragments to assign major fragment ids.
    Queue<Frag> q = Lists.newLinkedList();
    q.add(rootFrag);
    int majorFragmentId = 0;
    while (!q.isEmpty()) {
        Frag frag = q.remove();
        frag.majorFragmentId = majorFragmentId++;
        for (Frag child : frag) {
            q.add(child);
        }
    }
    // for each fragment, do a dfs of operators to assign operator ids.
    Map<Prel, OpId> ids = Maps.newIdentityHashMap();
    ids.put(rootFrag.root, new OpId(0, 0));
    for (Frag f : frags) {
        int id = 1;
        Queue<Prel> ops = Lists.newLinkedList();
        ops.add(f.root);
        while (!ops.isEmpty()) {
            Prel p = ops.remove();
            boolean isExchange = p instanceof ExchangePrel;
            if (p != f.root) {
                // we account for exchanges as receviers to guarantee unique identifiers.
                ids.put(p, new OpId(f.majorFragmentId, id++));
            }
            if (!isExchange || p == f.root) {
                List<Prel> children = Lists.reverse(Lists.newArrayList(p.iterator()));
                for (Prel child : children) {
                    ops.add(child);
                }
            }
        }
    }
    return ids;
}