Examples with MarkerSplitSource io.prestosql.snapshot.MarkerSplitSource used on opensource projects

Example 1 with MarkerSplitSource

use of io.prestosql.snapshot.MarkerSplitSource in project hetu-core by openlookeng.

the class DistributedExecutionPlanner method collectSources.

private List<MarkerSplitSource> collectSources(Map<PlanFragmentId, Object> leftmostSources, Object source) {
    if (source instanceof ValuesNode) {
        // TODO-cp-I2X9J6: should we worry about dependencies about Values operators, when it's the "left" of a join?
        return ImmutableList.of();
    }
    if (source instanceof RemoteSourceNode) {
        List<PlanFragmentId> fragments = ((RemoteSourceNode) source).getSourceFragmentIds();
        if (fragments.size() == 1) {
            return collectSources(leftmostSources, leftmostSources.get(fragments.get(0)));
        }
        List<MarkerSplitSource> sources = new ArrayList<>();
        for (PlanFragmentId id : fragments) {
            sources.addAll(collectSources(leftmostSources, leftmostSources.get(id)));
        }
        // Adding all these sources as "union dependencies" for each other, to make sure they produce the same set of markers.
        for (MarkerSplitSource unionSource : sources) {
            unionSource.addUnionSources(sources);
        }
        return sources;
    }
    // Must be a split source
    return ImmutableList.of((MarkerSplitSource) source);
}

Example 2 with MarkerSplitSource

use of io.prestosql.snapshot.MarkerSplitSource in project hetu-core by openlookeng.

the class DistributedExecutionPlanner method plan.

public StageExecutionPlan plan(SubPlan root, Session session, Mode mode, Long resumeSnapshotId, long nextSnapshotId) {
    ImmutableList.Builder<SplitSource> allSplitSources = ImmutableList.builder();
    try {
        if (mode != Mode.SNAPSHOT) {
            return doPlan(mode, root, session, resumeSnapshotId, nextSnapshotId, allSplitSources, null, null, null);
        }
        // Capture dependencies among table scan sources. Only need to do this for the initial planning.
        // The leftmost source of each fragment. Key is fragment id; value is SplitSource or ValuesNode or RemoteSourceNode
        Map<PlanFragmentId, Object> leftmostSources = new HashMap<>();
        // Source dependency. Key is SplitSource or ValuesNode or RemoteSourceNode; value is SplitSource or ValuesNode or RemoteSourceNode
        Multimap<Object, Object> sourceDependencies = HashMultimap.create();
        // List of sources from the same union. Values are SplitSource or ValuesNode or RemoteSourceNode.
        List<List<Object>> unionSources = new ArrayList<>();
        StageExecutionPlan ret = doPlan(mode, root, session, resumeSnapshotId, nextSnapshotId, allSplitSources, leftmostSources, sourceDependencies, unionSources);
        for (Map.Entry<Object, Object> entry : sourceDependencies.entries()) {
            List<MarkerSplitSource> right = collectSources(leftmostSources, entry.getValue());
            for (MarkerSplitSource source : collectSources(leftmostSources, entry.getKey())) {
                for (SplitSource dependency : right) {
                    source.addDependency((MarkerSplitSource) dependency);
                }
            }
        }
        List<MarkerSplitSource> sources = new ArrayList<>();
        for (List<Object> union : unionSources) {
            sources.clear();
            for (Object unionSource : union) {
                sources.addAll(collectSources(leftmostSources, unionSource));
            }
            // Adding all these sources as "union dependencies" for each other, to make sure they produce the same set of markers.
            for (MarkerSplitSource source : sources) {
                source.addUnionSources(sources);
            }
        }
        return ret;
    } catch (Throwable t) {
        allSplitSources.build().forEach(DistributedExecutionPlanner::closeSplitSource);
        throw t;
    }
}