Examples with RemoteSourceNode com.facebook.presto.sql.planner.plan.RemoteSourceNode used on opensource projects

Example 6 with RemoteSourceNode

use of com.facebook.presto.sql.planner.plan.RemoteSourceNode in project presto by prestodb.

the class SqlStageExecution method addExchangeLocations.

public synchronized void addExchangeLocations(PlanFragmentId fragmentId, Set<RemoteTask> sourceTasks, boolean noMoreExchangeLocations) {
    requireNonNull(fragmentId, "fragmentId is null");
    requireNonNull(sourceTasks, "sourceTasks is null");
    RemoteSourceNode remoteSource = exchangeSources.get(fragmentId);
    checkArgument(remoteSource != null, "Unknown remote source %s. Known sources are %s", fragmentId, exchangeSources.keySet());
    this.sourceTasks.putAll(remoteSource.getId(), sourceTasks);
    for (RemoteTask task : getAllTasks()) {
        ImmutableMultimap.Builder<PlanNodeId, Split> newSplits = ImmutableMultimap.builder();
        for (RemoteTask sourceTask : sourceTasks) {
            TaskStatus sourceTaskStatus = sourceTask.getTaskStatus();
            newSplits.put(remoteSource.getId(), createRemoteSplitFor(task.getTaskId(), sourceTask.getRemoteTaskLocation(), sourceTask.getTaskId()));
        }
        task.addSplits(newSplits.build());
    }
    if (noMoreExchangeLocations) {
        completeSourceFragments.add(fragmentId);
        // is the source now complete?
        if (completeSourceFragments.containsAll(remoteSource.getSourceFragmentIds())) {
            completeSources.add(remoteSource.getId());
            for (RemoteTask task : getAllTasks()) {
                task.noMoreSplits(remoteSource.getId());
            }
        }
    }
}

Example 7 with RemoteSourceNode

use of com.facebook.presto.sql.planner.plan.RemoteSourceNode in project presto by prestodb.

the class AllAtOnceExecutionSchedule method getPreferredScheduleOrder.

@VisibleForTesting
static List<PlanFragmentId> getPreferredScheduleOrder(Collection<PlanFragment> fragments) {
    // determine output fragment
    Set<PlanFragmentId> remoteSources = fragments.stream().map(PlanFragment::getRemoteSourceNodes).flatMap(Collection::stream).map(RemoteSourceNode::getSourceFragmentIds).flatMap(Collection::stream).collect(toImmutableSet());
    Set<PlanFragment> rootFragments = fragments.stream().filter(fragment -> !remoteSources.contains(fragment.getId())).collect(toImmutableSet());
    Visitor visitor = new Visitor(fragments);
    rootFragments.forEach(fragment -> visitor.processFragment(fragment.getId()));
    return visitor.getSchedulerOrder();
}

Example 8 with RemoteSourceNode

use of com.facebook.presto.sql.planner.plan.RemoteSourceNode in project presto by prestodb.

the class SectionExecutionFactory method createStreamingLinkedStageExecutions.

/**
 * returns a List of StageExecutionAndSchedulers in a postorder representation of the tree
 */
private List<StageExecutionAndScheduler> createStreamingLinkedStageExecutions(Session session, ExchangeLocationsConsumer parent, StreamingSubPlan plan, Function<PartitioningHandle, NodePartitionMap> partitioningCache, TableWriteInfo tableWriteInfo, Optional<SqlStageExecution> parentStageExecution, boolean summarizeTaskInfo, RemoteTaskFactory remoteTaskFactory, SplitSourceFactory splitSourceFactory, int attemptId) {
    ImmutableList.Builder<StageExecutionAndScheduler> stageExecutionAndSchedulers = ImmutableList.builder();
    PlanFragmentId fragmentId = plan.getFragment().getId();
    StageId stageId = new StageId(session.getQueryId(), fragmentId.getId());
    SqlStageExecution stageExecution = createSqlStageExecution(new StageExecutionId(stageId, attemptId), plan.getFragment(), remoteTaskFactory, session, summarizeTaskInfo, nodeTaskMap, executor, failureDetector, schedulerStats, tableWriteInfo);
    PartitioningHandle partitioningHandle = plan.getFragment().getPartitioning();
    List<RemoteSourceNode> remoteSourceNodes = plan.getFragment().getRemoteSourceNodes();
    Optional<int[]> bucketToPartition = getBucketToPartition(partitioningHandle, partitioningCache, plan.getFragment().getRoot(), remoteSourceNodes);
    // create child stages
    ImmutableSet.Builder<SqlStageExecution> childStagesBuilder = ImmutableSet.builder();
    for (StreamingSubPlan stagePlan : plan.getChildren()) {
        List<StageExecutionAndScheduler> subTree = createStreamingLinkedStageExecutions(session, stageExecution::addExchangeLocations, stagePlan.withBucketToPartition(bucketToPartition), partitioningCache, tableWriteInfo, Optional.of(stageExecution), summarizeTaskInfo, remoteTaskFactory, splitSourceFactory, attemptId);
        stageExecutionAndSchedulers.addAll(subTree);
        childStagesBuilder.add(getLast(subTree).getStageExecution());
    }
    Set<SqlStageExecution> childStageExecutions = childStagesBuilder.build();
    stageExecution.addStateChangeListener(newState -> {
        if (newState.isDone()) {
            childStageExecutions.forEach(SqlStageExecution::cancel);
        }
    });
    StageLinkage stageLinkage = new StageLinkage(fragmentId, parent, childStageExecutions);
    StageScheduler stageScheduler = createStageScheduler(splitSourceFactory, session, plan, partitioningCache, parentStageExecution, stageId, stageExecution, partitioningHandle, tableWriteInfo, childStageExecutions);
    stageExecutionAndSchedulers.add(new StageExecutionAndScheduler(stageExecution, stageLinkage, stageScheduler));
    return stageExecutionAndSchedulers.build();
}

Example 9 with RemoteSourceNode

use of com.facebook.presto.sql.planner.plan.RemoteSourceNode in project presto by prestodb.

the class TestPhasedExecutionSchedule method createJoinPlanFragment.

private static PlanFragment createJoinPlanFragment(JoinNode.Type joinType, String name, PlanFragment buildFragment, PlanFragment probeFragment) {
    RemoteSourceNode probe = new RemoteSourceNode(Optional.empty(), new PlanNodeId("probe_id"), probeFragment.getId(), ImmutableList.of(), false, Optional.empty(), REPARTITION);
    RemoteSourceNode build = new RemoteSourceNode(Optional.empty(), new PlanNodeId("build_id"), buildFragment.getId(), ImmutableList.of(), false, Optional.empty(), REPARTITION);
    PlanNode planNode = new JoinNode(Optional.empty(), new PlanNodeId(name + "_id"), joinType, probe, build, ImmutableList.of(), ImmutableList.<VariableReferenceExpression>builder().addAll(probe.getOutputVariables()).addAll(build.getOutputVariables()).build(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), ImmutableMap.of());
    return createFragment(planNode);
}

Example 10 with RemoteSourceNode

use of com.facebook.presto.sql.planner.plan.RemoteSourceNode in project presto by prestodb.

the class TestSourcePartitionedScheduler method createPlan.

private static SubPlan createPlan() {
    VariableReferenceExpression variable = new VariableReferenceExpression(Optional.empty(), "column", VARCHAR);
    // table scan with splitCount splits
    TableScanNode tableScan = new TableScanNode(Optional.empty(), TABLE_SCAN_NODE_ID, new TableHandle(CONNECTOR_ID, new TestingTableHandle(), TestingTransactionHandle.create(), Optional.empty()), ImmutableList.of(variable), ImmutableMap.of(variable, new TestingColumnHandle("column")), TupleDomain.all(), TupleDomain.all());
    RemoteSourceNode remote = new RemoteSourceNode(Optional.empty(), new PlanNodeId("remote_id"), new PlanFragmentId(0), ImmutableList.of(), false, Optional.empty(), GATHER);
    PlanFragment testFragment = new PlanFragment(new PlanFragmentId(0), new JoinNode(Optional.empty(), new PlanNodeId("join_id"), INNER, tableScan, remote, ImmutableList.of(), ImmutableList.<VariableReferenceExpression>builder().addAll(tableScan.getOutputVariables()).addAll(remote.getOutputVariables()).build(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), ImmutableMap.of()), ImmutableSet.of(variable), SOURCE_DISTRIBUTION, ImmutableList.of(TABLE_SCAN_NODE_ID), new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), ImmutableList.of(variable)), StageExecutionDescriptor.ungroupedExecution(), false, StatsAndCosts.empty(), Optional.empty());
    return new SubPlan(testFragment, ImmutableList.of());
}