Examples with MarkerPage io.prestosql.spi.snapshot.MarkerPage used on opensource projects

Example 41 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class TestArbitraryOutputBuffer method testMarkers.

@Test
public void testMarkers() {
    OutputBuffers outputBuffers = createInitialEmptyOutputBuffers(ARBITRARY).withBuffer(FIRST, BROADCAST_PARTITION_ID).withBuffer(SECOND, BROADCAST_PARTITION_ID);
    ArbitraryOutputBuffer buffer = createArbitraryBuffer(outputBuffers, sizeOfPages(5));
    ScheduledExecutorService scheduler = newScheduledThreadPool(4, daemonThreadsNamed("test-%s"));
    ScheduledExecutorService scheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed("test-scheduledExecutor-%s"));
    TaskContext taskContext = createTaskContext(scheduler, scheduledExecutor, TEST_SNAPSHOT_SESSION);
    buffer.setTaskContext(taskContext);
    buffer.addInputChannel("id");
    buffer.setNoMoreInputChannels();
    MarkerPage marker1 = MarkerPage.snapshotPage(1);
    MarkerPage marker2 = MarkerPage.snapshotPage(2);
    // add one item
    addPage(buffer, createPage(0));
    // broadcast 2 pages
    addPage(buffer, marker1, true);
    addPage(buffer, marker2, true);
    // target clients: ?, 1, 2, 1, 2
    // first client gets all 3 elements
    assertBufferResultEquals(TYPES, getBufferResult(buffer, FIRST, 0, sizeOfPages(3), NO_WAIT), bufferResult(0, createPage(0), marker1, marker2));
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getBufferedPages).collect(Collectors.toList()), Arrays.asList(3, 0));
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getPagesSent).collect(Collectors.toList()), Arrays.asList(0L, 0L));
    // acknowledge
    buffer.get(FIRST, 3, sizeOfPages(1)).cancel(true);
    assertQueueState(buffer, 2, FIRST, 0, 3);
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getBufferedPages).collect(Collectors.toList()), Arrays.asList(0, 0));
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getPagesSent).collect(Collectors.toList()), Arrays.asList(3L, 0L));
    // second client gets 2 elements
    assertBufferResultEquals(TYPES, getBufferResult(buffer, SECOND, 0, sizeOfPages(3), NO_WAIT), bufferResult(0, marker1, marker2));
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getBufferedPages).collect(Collectors.toList()), Arrays.asList(0, 2));
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getPagesSent).collect(Collectors.toList()), Arrays.asList(3L, 0L));
    // acknowledge
    buffer.get(SECOND, 2, sizeOfPages(1)).cancel(true);
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getBufferedPages).collect(Collectors.toList()), Arrays.asList(0, 0));
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getPagesSent).collect(Collectors.toList()), Arrays.asList(3L, 2L));
    // New buffer receives pending markers
    OutputBuffers outputBuffers1 = outputBuffers.withBuffer(THIRD, BROADCAST_PARTITION_ID);
    assertNotNull(outputBuffers1);
    buffer.setOutputBuffers(outputBuffers);
    assertBufferResultEquals(TYPES, getBufferResult(buffer, THIRD, 0, sizeOfPages(3), NO_WAIT), bufferResult(0, marker1, marker2));
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getBufferedPages).collect(Collectors.toList()), Arrays.asList(0, 0, 2));
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getPagesSent).collect(Collectors.toList()), Arrays.asList(3L, 2L, 0L));
    // acknowledge
    buffer.get(THIRD, 2, sizeOfPages(1)).cancel(true);
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getBufferedPages).collect(Collectors.toList()), Arrays.asList(0, 0, 0));
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getPagesSent).collect(Collectors.toList()), Arrays.asList(3L, 2L, 2L));
    // finish
    buffer.setNoMorePages();
    buffer.abort(FIRST);
    buffer.abort(SECOND);
    buffer.abort(THIRD);
    assertQueueClosed(buffer, 0, FIRST, 3);
    assertQueueClosed(buffer, 0, SECOND, 2);
    assertQueueClosed(buffer, 0, THIRD, 2);
    assertFinished(buffer);
}

Example 42 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class TestBroadcastOutputBuffer method testMarkers.

@Test
public void testMarkers() {
    OutputBuffers outputBuffers = createInitialEmptyOutputBuffers(BROADCAST).withBuffer(FIRST, BROADCAST_PARTITION_ID).withBuffer(SECOND, BROADCAST_PARTITION_ID).withNoMoreBufferIds();
    BroadcastOutputBuffer buffer = createBroadcastBuffer(outputBuffers, sizeOfPages(10));
    ScheduledExecutorService scheduler = newScheduledThreadPool(4, daemonThreadsNamed("test-%s"));
    ScheduledExecutorService scheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed("test-scheduledExecutor-%s"));
    TaskContext taskContext = createTaskContext(scheduler, scheduledExecutor, TEST_SNAPSHOT_SESSION);
    buffer.setTaskContext(taskContext);
    buffer.addInputChannel("id");
    buffer.setNoMoreInputChannels();
    MarkerPage marker1 = MarkerPage.snapshotPage(1);
    MarkerPage marker2 = MarkerPage.snapshotPage(2);
    // add one item
    addPage(buffer, createPage(0));
    // broadcast 2 pages
    addPage(buffer, marker1, true);
    addPage(buffer, marker2, true);
    // first client gets 2 elements
    assertBufferResultEquals(TYPES, getBufferResult(buffer, FIRST, 0, sizeOfPages(3), NO_WAIT), bufferResult(0, createPage(0), marker1, marker2));
    assertBufferResultEquals(TYPES, getBufferResult(buffer, SECOND, 0, sizeOfPages(3), NO_WAIT), bufferResult(0, createPage(0), marker1, marker2));
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getBufferedPages).collect(Collectors.toList()), Arrays.asList(3, 3));
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getPagesSent).collect(Collectors.toList()), Arrays.asList(0L, 0L));
    // acknowledge
    buffer.get(FIRST, 3, sizeOfPages(1)).cancel(true);
    assertQueueState(buffer, FIRST, 0, 3);
    buffer.get(SECOND, 3, sizeOfPages(1)).cancel(true);
    assertQueueState(buffer, SECOND, 0, 3);
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getBufferedPages).collect(Collectors.toList()), Arrays.asList(0, 0));
    assertEquals(buffer.getInfo().getBuffers().stream().map(BufferInfo::getPagesSent).collect(Collectors.toList()), Arrays.asList(3L, 3L));
    // finish
    buffer.setNoMorePages();
    buffer.abort(FIRST);
    buffer.abort(SECOND);
    assertQueueClosed(buffer, FIRST, 3);
    assertQueueClosed(buffer, SECOND, 3);
    assertFinished(buffer);
}

Example 43 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class PartitionedOutputOperator method addInput.

@Override
public void addInput(Page page) {
    requireNonNull(page, "page is null");
    if (page.getPositionCount() == 0) {
        return;
    }
    Page inputPage = page;
    if (inputPage instanceof MarkerPage) {
        // Send out all pending pages, then broadcast the marker. This must be done BEFORE snapshotState.processPage(),
        // otherwise what's in PageBuilders are captured, and their content will be sent AGAIN after resume.
        partitionFunction.flush(true);
        snapshotState.processPage(inputPage);
        MarkerPage marker = snapshotState.nextMarker();
        if (isStage0) {
            // Do not add marker to final output.
            return;
        }
        // The result is that for buffer #2, it receives marker 2 before marker 1.
        synchronized (partitionFunction.outputBuffer) {
            partitionFunction.outputBuffer.enqueue(0, Collections.singletonList(SerializedPage.forMarker(marker)), partitionFunction.id);
        }
    } else {
        inputPage = pagePreprocessor.apply(inputPage);
        partitionFunction.partitionPage(inputPage);
    }
    operatorContext.recordOutput(inputPage.getSizeInBytes(), inputPage.getPositionCount());
    // We use getSizeInBytes() here instead of getRetainedSizeInBytes() for an approximation of
    // the amount of memory used by the pageBuilders, because calculating the retained
    // size can be expensive especially for complex types.
    long partitionsSizeInBytes = partitionFunction.getSizeInBytes();
    // We also add partitionsInitialRetainedSize as an approximation of the object overhead of the partitions.
    systemMemoryContext.setBytes(partitionsSizeInBytes + partitionsInitialRetainedSize);
}

Example 44 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class TestExchangeOperator method testMarkers.

@Test
public void testMarkers() throws Exception {
    SourceOperator operator = createExchangeOperator(TEST_SNAPSHOT_SESSION);
    operator.addSplit(newRemoteSplit(TASK_1_ID));
    operator.addSplit(newRemoteSplit(TASK_2_ID));
    operator.noMoreSplits();
    MarkerPage marker = MarkerPage.snapshotPage(1);
    taskBuffers.getUnchecked(TASK_1_ID).addPages(3, false);
    taskBuffers.getUnchecked(TASK_2_ID).addPages(3, false);
    taskBuffers.getUnchecked(TASK_1_ID).addPage(marker, false);
    taskBuffers.getUnchecked(TASK_2_ID).addPages(3, false);
    taskBuffers.getUnchecked(TASK_2_ID).addPage(marker, false);
    taskBuffers.getUnchecked(TASK_2_ID).addPages(3, true);
    taskBuffers.getUnchecked(TASK_1_ID).addPages(3, true);
    // read the pages
    waitForPages(operator, 16);
    // wait for finished
    waitForFinished(operator);
}