Examples with RelMetadataQuery org.apache.calcite.rel.metadata.RelMetadataQuery used on opensource projects

Example 56 with RelMetadataQuery

use of org.apache.calcite.rel.metadata.RelMetadataQuery in project drill by apache.

the class NumberingRelWriter method explain_.

// ~ Methods ----------------------------------------------------------------
protected void explain_(RelNode rel, List<Pair<String, Object>> values) {
    RelMetadataQuery mq = RelMetadataQuery.instance();
    if (!mq.isVisibleInExplain(rel, detailLevel)) {
        // render children in place of this, at same level
        explainInputs(rel);
        return;
    }
    StringBuilder s = new StringBuilder();
    OpId id = ids.get(rel);
    if (id != null) {
        s.append(String.format("%02d-%02d", id.fragmentId, id.opId));
    } else {
        s.append("     ");
    }
    s.append("  ");
    if (id != null && id.opId == 0) {
        for (int i = 0; i < spacer.get(); i++) {
            s.append('-');
        }
    } else {
        spacer.spaces(s);
    }
    s.append("  ");
    s.append(rel.getRelTypeName().replace("Prel", ""));
    if (detailLevel != SqlExplainLevel.NO_ATTRIBUTES) {
        int j = 0;
        s.append(getDependentSrcOp(rel));
        for (Pair<String, Object> value : values) {
            if (value.right instanceof RelNode) {
                continue;
            }
            if (j++ == 0) {
                s.append("(");
            } else {
                s.append(", ");
            }
            s.append(value.left).append("=[").append(value.right).append("]");
        }
        if (j > 0) {
            s.append(")");
        }
    }
    if (detailLevel == SqlExplainLevel.ALL_ATTRIBUTES) {
        s.append(" : rowType = ").append(rel.getRowType()).append(": rowcount = ").append(mq.getRowCount(rel)).append(", cumulative cost = ").append(mq.getCumulativeCost(rel)).append(", id = ").append(rel.getId());
    }
    pw.println(s);
    spacer.add(2);
    explainInputs(rel);
    spacer.subtract(2);
}

Example 57 with RelMetadataQuery

use of org.apache.calcite.rel.metadata.RelMetadataQuery in project drill by apache.

the class RuntimeFilterVisitor method generateRuntimeFilter.

/**
 * Generate a possible RuntimeFilter of a HashJoinPrel, left some BF parameters of the generated RuntimeFilter
 * to be set later.
 *
 * @param hashJoinPrel
 * @return null or a partial information RuntimeFilterDef
 */
private RuntimeFilterDef generateRuntimeFilter(HashJoinPrel hashJoinPrel) {
    JoinRelType joinRelType = hashJoinPrel.getJoinType();
    JoinInfo joinInfo = hashJoinPrel.analyzeCondition();
    boolean allowJoin = (joinInfo.isEqui()) && (joinRelType == JoinRelType.INNER || joinRelType == JoinRelType.RIGHT);
    if (!allowJoin) {
        return null;
    }
    // TODO check whether to enable RuntimeFilter according to the NDV percent
    /**
     *     double threshold = 0.5;
     *     double percent = leftNDV / rightDNV;
     *     if (percent > threshold ) {
     *     return null;
     *     }
     */
    List<BloomFilterDef> bloomFilterDefs = new ArrayList<>();
    // find the possible left scan node of the left join key
    ScanPrel probeSideScanPrel = null;
    RelNode left = hashJoinPrel.getLeft();
    RelNode right = hashJoinPrel.getRight();
    ExchangePrel exchangePrel = findRightExchangePrel(right);
    if (exchangePrel == null) {
        // can only be BroadcastExchangePrel or HashToRandomExchangePrel
        return null;
    }
    List<String> leftFields = left.getRowType().getFieldNames();
    List<String> rightFields = right.getRowType().getFieldNames();
    List<Integer> leftKeys = hashJoinPrel.getLeftKeys();
    List<Integer> rightKeys = hashJoinPrel.getRightKeys();
    RelMetadataQuery metadataQuery = left.getCluster().getMetadataQuery();
    int i = 0;
    for (Integer leftKey : leftKeys) {
        String leftFieldName = leftFields.get(leftKey);
        Integer rightKey = rightKeys.get(i++);
        String rightFieldName = rightFields.get(rightKey);
        // This also avoids the left field of the join condition with a function call.
        ScanPrel scanPrel = findLeftScanPrel(leftFieldName, left);
        if (scanPrel != null) {
            boolean encounteredBlockNode = containBlockNode((Prel) left, scanPrel);
            if (encounteredBlockNode) {
                continue;
            }
            // Collect NDV from the Metadata
            RelDataType scanRowType = scanPrel.getRowType();
            RelDataTypeField field = scanRowType.getField(leftFieldName, true, true);
            int index = field.getIndex();
            Double ndv = metadataQuery.getDistinctRowCount(scanPrel, ImmutableBitSet.of(index), null);
            if (ndv == null) {
                // If NDV is not supplied, we use the row count to estimate the ndv.
                ndv = left.estimateRowCount(metadataQuery) * 0.1;
            }
            int bloomFilterSizeInBytes = BloomFilter.optimalNumOfBytes(ndv.longValue(), fpp);
            bloomFilterSizeInBytes = bloomFilterSizeInBytes > bloomFilterMaxSizeInBytesDef ? bloomFilterMaxSizeInBytesDef : bloomFilterSizeInBytes;
            // left the local parameter to be set later.
            BloomFilterDef bloomFilterDef = new BloomFilterDef(bloomFilterSizeInBytes, false, leftFieldName, rightFieldName);
            bloomFilterDef.setLeftNDV(ndv);
            bloomFilterDefs.add(bloomFilterDef);
            toAddRuntimeFilter.add(scanPrel);
            probeSideScanPrel = scanPrel;
        }
    }
    if (bloomFilterDefs.size() > 0) {
        // left sendToForeman parameter to be set later.
        RuntimeFilterDef runtimeFilterDef = new RuntimeFilterDef(true, false, bloomFilterDefs, false, -1);
        probeSideScan2hj.put(probeSideScanPrel, hashJoinPrel);
        return runtimeFilterDef;
    }
    return null;
}

Example 58 with RelMetadataQuery

use of org.apache.calcite.rel.metadata.RelMetadataQuery in project drill by apache.

the class PhysicalPlanCreator method getPrelCostEstimates.

private PrelCostEstimates getPrelCostEstimates(Prel originalPrel, PhysicalOperator op) {
    final RelMetadataQuery mq = originalPrel.getCluster().getMetadataQuery();
    final double estimatedRowCount = originalPrel.estimateRowCount(mq);
    final DrillCostBase costBase = (DrillCostBase) originalPrel.computeSelfCost(originalPrel.getCluster().getPlanner(), mq);
    final PrelCostEstimates costEstimates;
    if (!op.isBufferedOperator(context)) {
        costEstimates = new PrelCostEstimates(context.getOptions().getLong(ExecConstants.OUTPUT_BATCH_SIZE), estimatedRowCount);
    } else {
        costEstimates = new PrelCostEstimates(costBase.getMemory(), estimatedRowCount);
    }
    return costEstimates;
}

Example 59 with RelMetadataQuery

use of org.apache.calcite.rel.metadata.RelMetadataQuery in project flink by apache.

the class LogicalSnapshot method create.

/**
 * Creates a LogicalSnapshot.
 */
public static LogicalSnapshot create(RelNode input, RexNode period) {
    final RelOptCluster cluster = input.getCluster();
    final RelMetadataQuery mq = cluster.getMetadataQuery();
    final RelTraitSet traitSet = cluster.traitSet().replace(Convention.NONE).replaceIfs(RelCollationTraitDef.INSTANCE, () -> RelMdCollation.snapshot(mq, input)).replaceIf(RelDistributionTraitDef.INSTANCE, () -> RelMdDistribution.snapshot(mq, input));
    return new LogicalSnapshot(cluster, traitSet, input, period);
}

Example 60 with RelMetadataQuery

use of org.apache.calcite.rel.metadata.RelMetadataQuery in project flink by apache.

the class FlinkAggregateRemoveRule method matches.

@Override
public boolean matches(RelOptRuleCall call) {
    final Aggregate aggregate = call.rel(0);
    final RelNode input = call.rel(1);
    if (aggregate.getGroupCount() == 0 || aggregate.indicator || aggregate.getGroupType() != Aggregate.Group.SIMPLE) {
        return false;
    }
    for (AggregateCall aggCall : aggregate.getAggCallList()) {
        SqlKind aggCallKind = aggCall.getAggregation().getKind();
        // TODO supports more AggregateCalls
        boolean isAllowAggCall = aggCallKind == SqlKind.SUM || aggCallKind == SqlKind.MIN || aggCallKind == SqlKind.MAX || aggCall.getAggregation() instanceof SqlAuxiliaryGroupAggFunction;
        if (!isAllowAggCall || aggCall.filterArg >= 0 || aggCall.getArgList().size() != 1) {
            return false;
        }
    }
    final RelMetadataQuery mq = call.getMetadataQuery();
    return SqlFunctions.isTrue(mq.areColumnsUnique(input, aggregate.getGroupSet()));
}