Examples with BroadcastExchangePrel org.apache.drill.exec.planner.physical.BroadcastExchangePrel used on opensource projects

Example 1 with BroadcastExchangePrel

use of org.apache.drill.exec.planner.physical.BroadcastExchangePrel in project drill by apache.

the class RuntimeFilterVisitor method generateRuntimeFilter.

/**
 * Generate a possible RuntimeFilter of a HashJoinPrel, left some BF parameters of the generated RuntimeFilter
 * to be set later.
 *
 * @param hashJoinPrel
 * @return null or a partial information RuntimeFilterDef
 */
private RuntimeFilterDef generateRuntimeFilter(HashJoinPrel hashJoinPrel) {
    JoinRelType joinRelType = hashJoinPrel.getJoinType();
    JoinInfo joinInfo = hashJoinPrel.analyzeCondition();
    boolean allowJoin = (joinInfo.isEqui()) && (joinRelType == JoinRelType.INNER || joinRelType == JoinRelType.RIGHT);
    if (!allowJoin) {
        return null;
    }
    // TODO check whether to enable RuntimeFilter according to the NDV percent
    /**
     *     double threshold = 0.5;
     *     double percent = leftNDV / rightDNV;
     *     if (percent > threshold ) {
     *     return null;
     *     }
     */
    List<BloomFilterDef> bloomFilterDefs = new ArrayList<>();
    // find the possible left scan node of the left join key
    ScanPrel probeSideScanPrel = null;
    RelNode left = hashJoinPrel.getLeft();
    RelNode right = hashJoinPrel.getRight();
    ExchangePrel exchangePrel = findRightExchangePrel(right);
    if (exchangePrel == null) {
        // can only be BroadcastExchangePrel or HashToRandomExchangePrel
        return null;
    }
    List<String> leftFields = left.getRowType().getFieldNames();
    List<String> rightFields = right.getRowType().getFieldNames();
    List<Integer> leftKeys = hashJoinPrel.getLeftKeys();
    List<Integer> rightKeys = hashJoinPrel.getRightKeys();
    RelMetadataQuery metadataQuery = left.getCluster().getMetadataQuery();
    int i = 0;
    for (Integer leftKey : leftKeys) {
        String leftFieldName = leftFields.get(leftKey);
        Integer rightKey = rightKeys.get(i++);
        String rightFieldName = rightFields.get(rightKey);
        // This also avoids the left field of the join condition with a function call.
        ScanPrel scanPrel = findLeftScanPrel(leftFieldName, left);
        if (scanPrel != null) {
            boolean encounteredBlockNode = containBlockNode((Prel) left, scanPrel);
            if (encounteredBlockNode) {
                continue;
            }
            // Collect NDV from the Metadata
            RelDataType scanRowType = scanPrel.getRowType();
            RelDataTypeField field = scanRowType.getField(leftFieldName, true, true);
            int index = field.getIndex();
            Double ndv = metadataQuery.getDistinctRowCount(scanPrel, ImmutableBitSet.of(index), null);
            if (ndv == null) {
                // If NDV is not supplied, we use the row count to estimate the ndv.
                ndv = left.estimateRowCount(metadataQuery) * 0.1;
            }
            int bloomFilterSizeInBytes = BloomFilter.optimalNumOfBytes(ndv.longValue(), fpp);
            bloomFilterSizeInBytes = bloomFilterSizeInBytes > bloomFilterMaxSizeInBytesDef ? bloomFilterMaxSizeInBytesDef : bloomFilterSizeInBytes;
            // left the local parameter to be set later.
            BloomFilterDef bloomFilterDef = new BloomFilterDef(bloomFilterSizeInBytes, false, leftFieldName, rightFieldName);
            bloomFilterDef.setLeftNDV(ndv);
            bloomFilterDefs.add(bloomFilterDef);
            toAddRuntimeFilter.add(scanPrel);
            probeSideScanPrel = scanPrel;
        }
    }
    if (bloomFilterDefs.size() > 0) {
        // left sendToForeman parameter to be set later.
        RuntimeFilterDef runtimeFilterDef = new RuntimeFilterDef(true, false, bloomFilterDefs, false, -1);
        probeSideScan2hj.put(probeSideScanPrel, hashJoinPrel);
        return runtimeFilterDef;
    }
    return null;
}