Examples with RemoteTaskFactory com.facebook.presto.execution.RemoteTaskFactory used on opensource projects

Example 1 with RemoteTaskFactory

use of com.facebook.presto.execution.RemoteTaskFactory in project presto by prestodb.

the class LegacySqlQueryScheduler method updateStageExecutions.

/**
 * Utility function that rebuild a StreamingPlanSection, re-create stageExecutionAndScheduler for each of its stage, and finally update the stageExecutions map.
 */
private void updateStageExecutions(StreamingPlanSection section, Map<PlanFragment, PlanFragment> oldToNewFragment) {
    StreamingPlanSection newSection = new StreamingPlanSection(rewriteStreamingSubPlan(section.getPlan(), oldToNewFragment), section.getChildren());
    PlanFragment sectionRootFragment = newSection.getPlan().getFragment();
    Optional<int[]> bucketToPartition;
    OutputBuffers outputBuffers;
    ExchangeLocationsConsumer locationsConsumer;
    if (isRootFragment(sectionRootFragment)) {
        bucketToPartition = Optional.of(new int[1]);
        outputBuffers = createInitialEmptyOutputBuffers(sectionRootFragment.getPartitioningScheme().getPartitioning().getHandle()).withBuffer(new OutputBufferId(0), BROADCAST_PARTITION_ID).withNoMoreBufferIds();
        OutputBufferId rootBufferId = getOnlyElement(outputBuffers.getBuffers().keySet());
        locationsConsumer = (fragmentId, tasks, noMoreExchangeLocations) -> updateQueryOutputLocations(queryStateMachine, rootBufferId, tasks, noMoreExchangeLocations);
    } else {
        bucketToPartition = Optional.empty();
        outputBuffers = createDiscardingOutputBuffers();
        locationsConsumer = (fragmentId, tasks, noMoreExchangeLocations) -> {
        };
    }
    SectionExecution sectionExecution = sectionExecutionFactory.createSectionExecutions(session, newSection, locationsConsumer, bucketToPartition, outputBuffers, summarizeTaskInfo, remoteTaskFactory, splitSourceFactory, 0);
    addStateChangeListeners(sectionExecution);
    Map<StageId, StageExecutionAndScheduler> updatedStageExecutions = sectionExecution.getSectionStages().stream().collect(toImmutableMap(execution -> execution.getStageExecution().getStageExecutionId().getStageId(), identity()));
    synchronized (this) {
        stageExecutions.putAll(updatedStageExecutions);
    }
}

Example 2 with RemoteTaskFactory

use of com.facebook.presto.execution.RemoteTaskFactory in project presto by prestodb.

the class SectionExecutionFactory method createSectionExecutions.

/**
 * returns a List of SectionExecutions in a postorder representation of the tree
 */
public SectionExecution createSectionExecutions(Session session, StreamingPlanSection section, ExchangeLocationsConsumer locationsConsumer, Optional<int[]> bucketToPartition, OutputBuffers outputBuffers, boolean summarizeTaskInfo, RemoteTaskFactory remoteTaskFactory, SplitSourceFactory splitSourceFactory, int attemptId) {
    // Only fetch a distribution once per section to ensure all stages see the same machine assignments
    Map<PartitioningHandle, NodePartitionMap> partitioningCache = new HashMap<>();
    TableWriteInfo tableWriteInfo = createTableWriteInfo(section.getPlan(), metadata, session);
    List<StageExecutionAndScheduler> sectionStages = createStreamingLinkedStageExecutions(session, locationsConsumer, section.getPlan().withBucketToPartition(bucketToPartition), partitioningHandle -> partitioningCache.computeIfAbsent(partitioningHandle, handle -> nodePartitioningManager.getNodePartitioningMap(session, handle)), tableWriteInfo, Optional.empty(), summarizeTaskInfo, remoteTaskFactory, splitSourceFactory, attemptId);
    StageExecutionAndScheduler rootStage = getLast(sectionStages);
    rootStage.getStageExecution().setOutputBuffers(outputBuffers);
    return new SectionExecution(rootStage, sectionStages);
}