Examples with TableWriterMergeNode com.facebook.presto.sql.planner.plan.TableWriterMergeNode used on opensource projects

Example 1 with TableWriterMergeNode

use of com.facebook.presto.sql.planner.plan.TableWriterMergeNode in project presto by prestodb.

the class PlanFragmenter method analyzeGroupedExecution.

/*
     * In theory, recoverable grouped execution should be decided at query section level (i.e. a connected component of stages connected by remote exchanges).
     * This is because supporting mixed recoverable execution and non-recoverable execution within a query section adds unnecessary complications but provides little benefit,
     * because a single task failure is still likely to fail the non-recoverable stage.
     * However, since the concept of "query section" is not introduced until execution time as of now, it needs significant hacks to decide at fragmenting time.

     * TODO: We should introduce "query section" and make recoverability analysis done at query section level.
     */
private SubPlan analyzeGroupedExecution(Session session, SubPlan subPlan, boolean parentContainsTableFinish) {
    PlanFragment fragment = subPlan.getFragment();
    GroupedExecutionProperties properties = fragment.getRoot().accept(new GroupedExecutionTagger(session, metadata, nodePartitioningManager), null);
    if (properties.isSubTreeUseful()) {
        boolean preferDynamic = fragment.getRemoteSourceNodes().stream().allMatch(node -> node.getExchangeType() == REPLICATE);
        BucketNodeMap bucketNodeMap = nodePartitioningManager.getBucketNodeMap(session, fragment.getPartitioning(), preferDynamic);
        if (bucketNodeMap.isDynamic()) {
            /*
                 * We currently only support recoverable grouped execution if the following statements hold true:
                 *   - Current session enables recoverable grouped execution and table writer merge operator
                 *   - Parent sub plan contains TableFinishNode
                 *   - Current sub plan's root is TableWriterMergeNode or TableWriterNode
                 *   - Input connectors supports split source rewind
                 *   - Output connectors supports partition commit
                 *   - Bucket node map uses dynamic scheduling
                 *   - One table writer per task
                 */
            boolean recoverable = isRecoverableGroupedExecutionEnabled(session) && isTableWriterMergeOperatorEnabled(session) && parentContainsTableFinish && (fragment.getRoot() instanceof TableWriterMergeNode || fragment.getRoot() instanceof TableWriterNode) && properties.isRecoveryEligible();
            if (recoverable) {
                fragment = fragment.withRecoverableGroupedExecution(properties.getCapableTableScanNodes(), properties.getTotalLifespans());
            } else {
                fragment = fragment.withDynamicLifespanScheduleGroupedExecution(properties.getCapableTableScanNodes(), properties.getTotalLifespans());
            }
        } else {
            fragment = fragment.withFixedLifespanScheduleGroupedExecution(properties.getCapableTableScanNodes(), properties.getTotalLifespans());
        }
    }
    ImmutableList.Builder<SubPlan> result = ImmutableList.builder();
    boolean containsTableFinishNode = containsTableFinishNode(fragment);
    for (SubPlan child : subPlan.getChildren()) {
        result.add(analyzeGroupedExecution(session, child, containsTableFinishNode));
    }
    return new SubPlan(fragment, result.build());
}