Examples with OutputBufferId com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId used on opensource projects

Example 6 with OutputBufferId

use of com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId in project presto by prestodb.

the class SqlQueryScheduler method createStageExecutions.

private List<SectionExecution> createStageExecutions(List<StreamingPlanSection> sections) {
    ImmutableList.Builder<SectionExecution> result = ImmutableList.builder();
    for (StreamingPlanSection section : sections) {
        StageId sectionId = getStageId(section.getPlan().getFragment().getId());
        List<SectionExecution> attempts = sectionExecutions.computeIfAbsent(sectionId, (ignored) -> new CopyOnWriteArrayList<>());
        // sectionExecutions only get created when they are about to be scheduled, so there should
        // never be a non-failed SectionExecution for a section that's ready for execution
        verify(attempts.isEmpty() || getLast(attempts).isFailed(), "Non-failed sectionExecutions already exists");
        PlanFragment sectionRootFragment = section.getPlan().getFragment();
        Optional<int[]> bucketToPartition;
        OutputBuffers outputBuffers;
        ExchangeLocationsConsumer locationsConsumer;
        if (isRootFragment(sectionRootFragment)) {
            bucketToPartition = Optional.of(new int[1]);
            outputBuffers = createInitialEmptyOutputBuffers(sectionRootFragment.getPartitioningScheme().getPartitioning().getHandle()).withBuffer(new OutputBufferId(0), BROADCAST_PARTITION_ID).withNoMoreBufferIds();
            OutputBufferId rootBufferId = getOnlyElement(outputBuffers.getBuffers().keySet());
            locationsConsumer = (fragmentId, tasks, noMoreExchangeLocations) -> updateQueryOutputLocations(queryStateMachine, rootBufferId, tasks, noMoreExchangeLocations);
        } else {
            bucketToPartition = Optional.empty();
            outputBuffers = createDiscardingOutputBuffers();
            locationsConsumer = (fragmentId, tasks, noMoreExchangeLocations) -> {
            };
        }
        int attemptId = attempts.size();
        SectionExecution sectionExecution = sectionExecutionFactory.createSectionExecutions(session, section, locationsConsumer, bucketToPartition, outputBuffers, summarizeTaskInfo, remoteTaskFactory, splitSourceFactory, attemptId);
        addStateChangeListeners(sectionExecution);
        attempts.add(sectionExecution);
        result.add(sectionExecution);
    }
    return result.build();
}

Example 7 with OutputBufferId

use of com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId in project presto by prestodb.

the class TestArbitraryOutputBuffer method testResumeFromPreviousPosition.

@Test
public void testResumeFromPreviousPosition() {
    OutputBuffers outputBuffers = createInitialEmptyOutputBuffers(ARBITRARY);
    OutputBufferId[] ids = new OutputBufferId[5];
    for (int i = 0; i < ids.length; i++) {
        ids[i] = new OutputBufferId(i);
        outputBuffers = outputBuffers.withBuffer(ids[i], i);
    }
    ArbitraryOutputBuffer buffer = createArbitraryBuffer(outputBuffers, sizeOfPages(5));
    assertFalse(buffer.isFinished());
    Map<OutputBufferId, ListenableFuture<BufferResult>> firstReads = new HashMap<>();
    for (OutputBufferId id : ids) {
        firstReads.put(id, buffer.get(id, 0L, sizeOfPages(1)));
    }
    // All must be blocked initially
    assertThat(firstReads.values()).allMatch(future -> !future.isDone());
    List<ListenableFuture<BufferResult>> secondReads = new ArrayList<>();
    for (int i = 0; i < ids.length; i++) {
        // add one page
        addPage(buffer, createPage(33));
        assertThat(secondReads).allMatch(future -> !future.isDone(), "No secondary reads should complete until after all first reads");
        List<OutputBufferId> completedIds = firstReads.entrySet().stream().filter(entry -> entry.getValue().isDone()).map(Map.Entry::getKey).collect(toList());
        assertEquals(completedIds.size(), 1, "One completed buffer read per page addition");
        OutputBufferId completed = completedIds.get(0);
        BufferResult result = getFuture(firstReads.remove(completed), NO_WAIT);
        // Store completion order of first for follow up sequence
        secondReads.add(buffer.get(completed, result.getNextToken(), sizeOfPages(1)));
    }
    // Test sanity
    assertEquals(secondReads.size(), ids.length);
    // Completion order should be identical to the first iteration at this point
    for (int i = 0; i < ids.length; i++) {
        // add one page
        addPage(buffer, createPage(33));
        assertTrue(secondReads.get(i).isDone(), "Invalid second read completion order at index: " + i);
    }
}

Example 8 with OutputBufferId

use of com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId in project presto by prestodb.

the class TestBroadcastOutputBufferManager method test.

@Test
public void test() {
    AtomicReference<OutputBuffers> outputBufferTarget = new AtomicReference<>();
    BroadcastOutputBufferManager hashOutputBufferManager = new BroadcastOutputBufferManager(outputBufferTarget::set);
    assertEquals(outputBufferTarget.get(), createInitialEmptyOutputBuffers(BROADCAST));
    hashOutputBufferManager.addOutputBuffers(ImmutableList.of(new OutputBufferId(0)), false);
    OutputBuffers expectedOutputBuffers = createInitialEmptyOutputBuffers(BROADCAST).withBuffer(new OutputBufferId(0), BROADCAST_PARTITION_ID);
    assertEquals(outputBufferTarget.get(), expectedOutputBuffers);
    hashOutputBufferManager.addOutputBuffers(ImmutableList.of(new OutputBufferId(1), new OutputBufferId(2)), false);
    expectedOutputBuffers = expectedOutputBuffers.withBuffer(new OutputBufferId(1), BROADCAST_PARTITION_ID);
    expectedOutputBuffers = expectedOutputBuffers.withBuffer(new OutputBufferId(2), BROADCAST_PARTITION_ID);
    assertEquals(outputBufferTarget.get(), expectedOutputBuffers);
    // set no more buffers
    hashOutputBufferManager.addOutputBuffers(ImmutableList.of(new OutputBufferId(3)), true);
    expectedOutputBuffers = expectedOutputBuffers.withBuffer(new OutputBufferId(3), BROADCAST_PARTITION_ID);
    expectedOutputBuffers = expectedOutputBuffers.withNoMoreBufferIds();
    assertEquals(outputBufferTarget.get(), expectedOutputBuffers);
    // try to add another buffer, which should not result in an error
    // and output buffers should not change
    hashOutputBufferManager.addOutputBuffers(ImmutableList.of(new OutputBufferId(5)), false);
    assertEquals(outputBufferTarget.get(), expectedOutputBuffers);
    // try to set no more buffers again, which should not result in an error
    // and output buffers should not change
    hashOutputBufferManager.addOutputBuffers(ImmutableList.of(new OutputBufferId(6)), true);
    assertEquals(outputBufferTarget.get(), expectedOutputBuffers);
}

Example 9 with OutputBufferId

use of com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId in project presto by prestodb.

the class AsyncPageTransportServlet method parseURI.

protected void parseURI(String requestURI, HttpServletRequest request, HttpServletResponse response) throws IOException {
    // Split a task URI without allocating a list and unnecessary strings
    // Example:  /v1/task/async/{taskId}/results/{bufferId}/{token}
    TaskId taskId = null;
    OutputBufferId bufferId = null;
    long token = 0;
    int previousIndex = -1;
    for (int part = 0; part < 8; part++) {
        int nextIndex = requestURI.indexOf('/', previousIndex + 1);
        if (nextIndex == -1 && part != 7 || nextIndex != -1 && part == 7) {
            reportFailure(response, format("Unexpected URI for task result request in async mode: %s", requestURI));
            return;
        }
        switch(part) {
            case 4:
                taskId = TaskId.valueOf(requestURI.substring(previousIndex + 1, nextIndex));
                break;
            case 6:
                bufferId = OutputBufferId.fromString(requestURI.substring(previousIndex + 1, nextIndex));
                break;
            case 7:
                token = parseLong(requestURI.substring(previousIndex + 1));
                break;
        }
        previousIndex = nextIndex;
    }
    // This is sent forward instead of returned to avoid allocations
    processRequest(requestURI, taskId, bufferId, token, request, response);
}

Example 10 with OutputBufferId

use of com.facebook.presto.execution.buffer.OutputBuffers.OutputBufferId in project presto by prestodb.

the class TestThriftTaskIntegration method testServer.

@Test
public void testServer() {
    AddressSelector<SimpleAddressSelector.SimpleAddress> addressSelector = new SimpleAddressSelector(ImmutableSet.of(HostAndPort.fromParts("localhost", thriftServerPort)), true);
    try (DriftNettyMethodInvokerFactory<?> invokerFactory = createStaticDriftNettyMethodInvokerFactory(new DriftNettyClientConfig())) {
        DriftClientFactory clientFactory = new DriftClientFactory(new ThriftCodecManager(), invokerFactory, addressSelector, NORMAL_RESULT);
        ThriftTaskClient client = clientFactory.createDriftClient(ThriftTaskClient.class).get();
        // get buffer result
        ListenableFuture<ThriftBufferResult> result = client.getResults(TaskId.valueOf("queryid.0.0.0"), new OutputBufferId(1), 0, 100);
        assertTrue(result.get().isBufferComplete());
        assertTrue(result.get().getSerializedPages().isEmpty());
        assertEquals(result.get().getToken(), 1);
        assertEquals(result.get().getTaskInstanceId(), "test");
        // ack buffer result
        // sync
        client.acknowledgeResults(TaskId.valueOf("queryid.0.0.0"), new OutputBufferId(1), 42).get();
        // fire and forget
        client.acknowledgeResults(TaskId.valueOf("queryid.0.0.0"), new OutputBufferId(1), 42);
        // abort buffer result
        client.abortResults(TaskId.valueOf("queryid.0.0.0"), new OutputBufferId(1)).get();
    } catch (Exception e) {
        fail();
    }
}