Examples with OutputBuffers com.facebook.presto.execution.buffer.OutputBuffers used on opensource projects

Example 6 with OutputBuffers

use of com.facebook.presto.execution.buffer.OutputBuffers in project presto by prestodb.

the class TestOptimizedPartitionedOutputOperator method createPartitionedOutputBuffer.

private TestingPartitionedOutputBuffer createPartitionedOutputBuffer() {
    OutputBuffers buffers = createInitialEmptyOutputBuffers(PARTITIONED);
    for (int partition = 0; partition < PARTITION_COUNT; partition++) {
        buffers = buffers.withBuffer(new OutputBuffers.OutputBufferId(partition), partition);
    }
    TestingPartitionedOutputBuffer buffer = createPartitionedBuffer(buffers.withNoMoreBufferIds(), // don't let output buffer block
    new DataSize(Long.MAX_VALUE, BYTE));
    buffer.registerLifespanCompletionCallback(ignore -> {
    });
    return buffer;
}

Example 7 with OutputBuffers

use of com.facebook.presto.execution.buffer.OutputBuffers in project presto by prestodb.

the class TestSqlTask method testBufferCloseOnFinish.

@Test
public void testBufferCloseOnFinish() throws Exception {
    SqlTask sqlTask = createInitialTask();
    OutputBuffers outputBuffers = createInitialEmptyOutputBuffers(PARTITIONED).withBuffer(OUT, 0).withNoMoreBufferIds();
    updateTask(sqlTask, EMPTY_SOURCES, outputBuffers);
    ListenableFuture<BufferResult> bufferResult = sqlTask.getTaskResults(OUT, 0, new DataSize(1, MEGABYTE));
    assertFalse(bufferResult.isDone());
    // close the sources (no splits will ever be added)
    updateTask(sqlTask, ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(), true)), outputBuffers);
    // finish the task by calling abort on it
    sqlTask.abortTaskResults(OUT);
    // buffer will be closed by cancel event (wait for event to fire)
    bufferResult.get(1, SECONDS);
    // verify the buffer is closed
    bufferResult = sqlTask.getTaskResults(OUT, 0, new DataSize(1, MEGABYTE));
    assertTrue(bufferResult.isDone());
    assertTrue(bufferResult.get().isBufferComplete());
}

Example 8 with OutputBuffers

use of com.facebook.presto.execution.buffer.OutputBuffers in project presto by prestodb.

the class SectionExecutionFactory method createSectionExecutions.

/**
 * returns a List of SectionExecutions in a postorder representation of the tree
 */
public SectionExecution createSectionExecutions(Session session, StreamingPlanSection section, ExchangeLocationsConsumer locationsConsumer, Optional<int[]> bucketToPartition, OutputBuffers outputBuffers, boolean summarizeTaskInfo, RemoteTaskFactory remoteTaskFactory, SplitSourceFactory splitSourceFactory, int attemptId) {
    // Only fetch a distribution once per section to ensure all stages see the same machine assignments
    Map<PartitioningHandle, NodePartitionMap> partitioningCache = new HashMap<>();
    TableWriteInfo tableWriteInfo = createTableWriteInfo(section.getPlan(), metadata, session);
    List<StageExecutionAndScheduler> sectionStages = createStreamingLinkedStageExecutions(session, locationsConsumer, section.getPlan().withBucketToPartition(bucketToPartition), partitioningHandle -> partitioningCache.computeIfAbsent(partitioningHandle, handle -> nodePartitioningManager.getNodePartitioningMap(session, handle)), tableWriteInfo, Optional.empty(), summarizeTaskInfo, remoteTaskFactory, splitSourceFactory, attemptId);
    StageExecutionAndScheduler rootStage = getLast(sectionStages);
    rootStage.getStageExecution().setOutputBuffers(outputBuffers);
    return new SectionExecution(rootStage, sectionStages);
}

Example 9 with OutputBuffers

use of com.facebook.presto.execution.buffer.OutputBuffers in project presto by prestodb.

the class SqlQueryScheduler method createStageExecutions.

private List<SectionExecution> createStageExecutions(List<StreamingPlanSection> sections) {
    ImmutableList.Builder<SectionExecution> result = ImmutableList.builder();
    for (StreamingPlanSection section : sections) {
        StageId sectionId = getStageId(section.getPlan().getFragment().getId());
        List<SectionExecution> attempts = sectionExecutions.computeIfAbsent(sectionId, (ignored) -> new CopyOnWriteArrayList<>());
        // sectionExecutions only get created when they are about to be scheduled, so there should
        // never be a non-failed SectionExecution for a section that's ready for execution
        verify(attempts.isEmpty() || getLast(attempts).isFailed(), "Non-failed sectionExecutions already exists");
        PlanFragment sectionRootFragment = section.getPlan().getFragment();
        Optional<int[]> bucketToPartition;
        OutputBuffers outputBuffers;
        ExchangeLocationsConsumer locationsConsumer;
        if (isRootFragment(sectionRootFragment)) {
            bucketToPartition = Optional.of(new int[1]);
            outputBuffers = createInitialEmptyOutputBuffers(sectionRootFragment.getPartitioningScheme().getPartitioning().getHandle()).withBuffer(new OutputBufferId(0), BROADCAST_PARTITION_ID).withNoMoreBufferIds();
            OutputBufferId rootBufferId = getOnlyElement(outputBuffers.getBuffers().keySet());
            locationsConsumer = (fragmentId, tasks, noMoreExchangeLocations) -> updateQueryOutputLocations(queryStateMachine, rootBufferId, tasks, noMoreExchangeLocations);
        } else {
            bucketToPartition = Optional.empty();
            outputBuffers = createDiscardingOutputBuffers();
            locationsConsumer = (fragmentId, tasks, noMoreExchangeLocations) -> {
            };
        }
        int attemptId = attempts.size();
        SectionExecution sectionExecution = sectionExecutionFactory.createSectionExecutions(session, section, locationsConsumer, bucketToPartition, outputBuffers, summarizeTaskInfo, remoteTaskFactory, splitSourceFactory, attemptId);
        addStateChangeListeners(sectionExecution);
        attempts.add(sectionExecution);
        result.add(sectionExecution);
    }
    return result.build();
}

Example 10 with OutputBuffers

use of com.facebook.presto.execution.buffer.OutputBuffers in project presto by prestodb.

the class TestBroadcastOutputBufferManager method test.

@Test
public void test() {
    AtomicReference<OutputBuffers> outputBufferTarget = new AtomicReference<>();
    BroadcastOutputBufferManager hashOutputBufferManager = new BroadcastOutputBufferManager(outputBufferTarget::set);
    assertEquals(outputBufferTarget.get(), createInitialEmptyOutputBuffers(BROADCAST));
    hashOutputBufferManager.addOutputBuffers(ImmutableList.of(new OutputBufferId(0)), false);
    OutputBuffers expectedOutputBuffers = createInitialEmptyOutputBuffers(BROADCAST).withBuffer(new OutputBufferId(0), BROADCAST_PARTITION_ID);
    assertEquals(outputBufferTarget.get(), expectedOutputBuffers);
    hashOutputBufferManager.addOutputBuffers(ImmutableList.of(new OutputBufferId(1), new OutputBufferId(2)), false);
    expectedOutputBuffers = expectedOutputBuffers.withBuffer(new OutputBufferId(1), BROADCAST_PARTITION_ID);
    expectedOutputBuffers = expectedOutputBuffers.withBuffer(new OutputBufferId(2), BROADCAST_PARTITION_ID);
    assertEquals(outputBufferTarget.get(), expectedOutputBuffers);
    // set no more buffers
    hashOutputBufferManager.addOutputBuffers(ImmutableList.of(new OutputBufferId(3)), true);
    expectedOutputBuffers = expectedOutputBuffers.withBuffer(new OutputBufferId(3), BROADCAST_PARTITION_ID);
    expectedOutputBuffers = expectedOutputBuffers.withNoMoreBufferIds();
    assertEquals(outputBufferTarget.get(), expectedOutputBuffers);
    // try to add another buffer, which should not result in an error
    // and output buffers should not change
    hashOutputBufferManager.addOutputBuffers(ImmutableList.of(new OutputBufferId(5)), false);
    assertEquals(outputBufferTarget.get(), expectedOutputBuffers);
    // try to set no more buffers again, which should not result in an error
    // and output buffers should not change
    hashOutputBufferManager.addOutputBuffers(ImmutableList.of(new OutputBufferId(6)), true);
    assertEquals(outputBufferTarget.get(), expectedOutputBuffers);
}