Examples with MarkerPage io.prestosql.spi.snapshot.MarkerPage used on opensource projects

Example 26 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class TestLocalExchange method testMarkerBroadcast.

@Test(dataProvider = "markerExecutions")
public void testMarkerBroadcast(PartitioningHandle partitioningHandle, PipelineExecutionStrategy executionStrategy) {
    List<Integer> partitionChannels = partitioningHandle == FIXED_HASH_DISTRIBUTION ? ImmutableList.of(0) : ImmutableList.of();
    LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory(partitioningHandle, 2, TYPES, partitionChannels, Optional.empty(), executionStrategy, LOCAL_EXCHANGE_MAX_BUFFERED_BYTES);
    LocalExchangeSinkFactoryId localExchangeSinkFactoryId = localExchangeFactory.newSinkFactoryId();
    localExchangeFactory.noMoreSinkFactories();
    run(localExchangeFactory, executionStrategy, exchange -> {
        assertEquals(exchange.getBufferCount(), 2);
        assertEquals(exchange.getBufferedBytes(), 0);
        final String sinkAId = "sinkA";
        final String sinkBId = "sinkB";
        LocalExchangeSinkFactory sinkFactory = exchange.getSinkFactory(localExchangeSinkFactoryId);
        LocalExchangeSink sinkA = sinkFactory.createSink(sinkAId);
        assertSinkCanWrite(sinkA);
        LocalExchangeSink sinkB = sinkFactory.createSink(sinkBId);
        assertSinkCanWrite(sinkB);
        sinkFactory.close();
        sinkFactory.noMoreSinkFactories();
        LocalExchangeSource sourceA = exchange.getSource(0);
        assertSource(sourceA, 0);
        LocalExchangeSource sourceB = exchange.getSource(1);
        assertSource(sourceB, 0);
        MarkerPage marker1 = MarkerPage.snapshotPage(1);
        MarkerPage marker2 = MarkerPage.snapshotPage(2);
        MarkerPage resume1 = MarkerPage.resumePage(1);
        long markerSize = marker1.getRetainedSizeInBytes();
        sinkA.addPage(marker1, sinkAId);
        assertSource(sourceA, 1);
        assertSource(sourceB, 1);
        assertEquals(exchange.getBufferedBytes(), markerSize * 2);
        sinkB.addPage(marker2, sinkBId);
        assertSource(sourceA, 2);
        assertSource(sourceB, 2);
        assertEquals(exchange.getBufferedBytes(), markerSize * 4);
        assertRemovePage(sourceA, marker1, sinkAId);
        assertSource(sourceA, 1);
        assertSource(sourceB, 2);
        assertEquals(exchange.getBufferedBytes(), markerSize * 3);
        assertRemovePage(sourceA, marker2, sinkBId);
        assertSource(sourceA, 0);
        assertSource(sourceB, 2);
        assertEquals(exchange.getBufferedBytes(), markerSize * 2);
        sinkA.finish();
        assertSinkFinished(sinkA);
        assertEquals(exchange.getBufferedBytes(), markerSize * 2);
        sinkB.addPage(resume1, sinkBId);
        assertSource(sourceA, 1);
        assertSource(sourceB, 3);
        assertEquals(exchange.getBufferedBytes(), markerSize * 4);
        sinkB.finish();
        assertSinkFinished(sinkB);
        assertSource(sourceA, 1);
        assertSource(sourceB, 3);
        assertEquals(exchange.getBufferedBytes(), markerSize * 4);
        assertRemovePage(sourceA, resume1, sinkBId);
        assertSourceFinished(sourceA);
        assertSource(sourceB, 3);
        assertEquals(exchange.getBufferedBytes(), markerSize * 3);
        assertRemovePage(sourceB, marker1, sinkAId);
        assertRemovePage(sourceB, marker2, sinkBId);
        assertSourceFinished(sourceA);
        assertSource(sourceB, 1);
        assertEquals(exchange.getBufferedBytes(), markerSize * 1);
        assertRemovePage(sourceB, resume1, sinkBId);
        assertSourceFinished(sourceA);
        assertSourceFinished(sourceB);
        assertEquals(exchange.getBufferedBytes(), markerSize * 0);
    });
}

Example 27 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class TestValuesOperator method testNormal.

@Test
public void testNormal() {
    List<Page> pages = ImmutableList.of(new Page(1));
    ValuesOperator operator = new ValuesOperator(mockOperatorContext(false), pages, 0);
    Page page = operator.getOutput();
    assertNotNull(page);
    assertFalse(page instanceof MarkerPage);
    page = operator.getOutput();
    assertNull(page);
}

Example 28 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class TestValuesOperator method testSnapshotPeekMarker.

@Test
public void testSnapshotPeekMarker() {
    List<Page> pages = ImmutableList.of(new Page(1), MarkerPage.snapshotPage(1));
    ValuesOperator operator = new ValuesOperator(mockOperatorContext(true), pages, 0);
    Page page = operator.pollMarker();
    assertNull(page);
    page = operator.getOutput();
    assertNotNull(page);
    assertFalse(page instanceof MarkerPage);
    page = operator.pollMarker();
    assertTrue(page instanceof MarkerPage);
    page = operator.pollMarker();
    assertNull(page);
    page = operator.getOutput();
    assertNull(page);
}

Example 29 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class TestValuesOperator method testResumePeekMarker.

@Test
public void testResumePeekMarker() {
    List<Page> pages = ImmutableList.of(new Page(1), MarkerPage.resumePage(1), MarkerPage.snapshotPage(2));
    ValuesOperator operator = new ValuesOperator(mockOperatorContext(true), pages, 1);
    Page page = operator.pollMarker();
    assertTrue(page instanceof MarkerPage && ((MarkerPage) page).isResuming());
    page = operator.pollMarker();
    assertTrue(page instanceof MarkerPage && !((MarkerPage) page).isResuming());
    page = operator.pollMarker();
    assertNull(page);
    page = operator.getOutput();
    assertNull(page);
}

Example 30 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class TestWindowOperator method testCaptureRestoreWithSpill.

@Test
public void testCaptureRestoreWithSpill() throws Exception {
    // Initialization
    Path spillPath = Paths.get("/tmp/hetu/snapshot/");
    GenericSpillerFactory genericSpillerFactory = createGenericSpillerFactory(spillPath, fileSystemClientManager, false, null);
    SnapshotConfig snapshotConfig = new SnapshotConfig();
    snapshotUtils = new SnapshotUtils(fileSystemClientManager, snapshotConfig, new InMemoryNodeManager());
    snapshotUtils.initialize();
    ImmutableList.Builder<Page> outputPages = ImmutableList.builder();
    List<Page> input1 = rowPagesBuilder(VARCHAR, BIGINT, DOUBLE, BOOLEAN).row("b", -1L, -0.1, true).row("a", 2L, 0.3, false).row("a", 4L, 0.2, true).pageBreak().row("b", 5L, 0.4, false).row("a", 6L, 0.1, true).build();
    List<Page> input2 = rowPagesBuilder(VARCHAR, BIGINT, DOUBLE, BOOLEAN).row("c", -1L, -0.1, true).row("d", 2L, 0.3, false).row("c", 4L, 0.2, true).pageBreak().row("d", 5L, 0.4, false).build();
    WindowOperatorFactory operatorFactory = new WindowOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN), Ints.asList(0, 1, 2, 3), ROW_NUMBER, Ints.asList(0), ImmutableList.of(), Ints.asList(1), ImmutableList.copyOf(new SortOrder[] { SortOrder.ASC_NULLS_LAST }), 0, 10, new PagesIndex.TestingFactory(false), true, genericSpillerFactory, new OrderingCompiler());
    DriverContext driverContext = createDriverContext(defaultMemoryLimit, TEST_SNAPSHOT_SESSION);
    WindowOperator windowOperator = (WindowOperator) operatorFactory.createOperator(driverContext);
    // Step1: add the first 2 pages
    for (Page page : input1) {
        windowOperator.addInput(page);
        windowOperator.getOutput();
    }
    // Step2: spilling happened here
    getFutureValue(windowOperator.startMemoryRevoke());
    windowOperator.finishMemoryRevoke();
    // Step3: add a marker page to make 'capture1' happened
    MarkerPage marker = MarkerPage.snapshotPage(1);
    windowOperator.addInput(marker);
    windowOperator.getOutput();
    // Step4: add another 2 pages
    for (Page page : input2) {
        windowOperator.addInput(page);
        windowOperator.getOutput();
    }
    // Step5: assume the task is rescheduled due to failure and everything is re-constructed
    driverContext = createDriverContext(8, TEST_SNAPSHOT_SESSION);
    operatorFactory = new WindowOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN), Ints.asList(0, 1, 2, 3), ROW_NUMBER, Ints.asList(0), ImmutableList.of(), Ints.asList(1), ImmutableList.copyOf(new SortOrder[] { SortOrder.ASC_NULLS_LAST }), 0, 10, new PagesIndex.TestingFactory(false), true, genericSpillerFactory, new OrderingCompiler());
    windowOperator = (WindowOperator) operatorFactory.createOperator(driverContext);
    // Step6: restore to 'capture1', the spiller should contains the reference of the first 2 pages for now.
    MarkerPage resumeMarker = MarkerPage.resumePage(1);
    windowOperator.addInput(resumeMarker);
    windowOperator.getOutput();
    // Step7: continue to add another 2 pages
    for (Page page : input2) {
        windowOperator.addInput(page);
        windowOperator.getOutput();
    }
    windowOperator.finish();
    // Compare the results
    MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, BIGINT, DOUBLE, BOOLEAN, BIGINT).row("a", 2L, 0.3, false, 1L).row("a", 4L, 0.2, true, 2L).row("a", 6L, 0.1, true, 3L).row("b", -1L, -0.1, true, 1L).row("b", 5L, 0.4, false, 2L).row("c", -1L, -0.1, true, 1L).row("c", 4L, 0.2, true, 2L).row("d", 2L, 0.3, false, 1L).row("d", 5L, 0.4, false, 2L).build();
    Page p = windowOperator.getOutput();
    while (p == null) {
        p = windowOperator.getOutput();
    }
    outputPages.add(p);
    MaterializedResult actual = toMaterializedResult(driverContext.getSession(), expected.getTypes(), outputPages.build());
    Assert.assertEquals(actual, expected);
}