Examples with MarkerPage io.prestosql.spi.snapshot.MarkerPage used on opensource projects

Example 6 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class TestWindowOperator method testCaptureRestoreWithoutSpill.

@Test
public void testCaptureRestoreWithoutSpill() {
    SnapshotConfig snapshotConfig = new SnapshotConfig();
    snapshotUtils = new SnapshotUtils(fileSystemClientManager, snapshotConfig, new InMemoryNodeManager());
    snapshotUtils.initialize();
    ImmutableList.Builder<Page> outputPages = ImmutableList.builder();
    List<Page> input1 = rowPagesBuilder(VARCHAR, BIGINT, DOUBLE, BOOLEAN).row("b", -1L, -0.1, true).row("a", 2L, 0.3, false).row("a", 4L, 0.2, true).pageBreak().row("b", 5L, 0.4, false).row("a", 6L, 0.1, true).build();
    List<Page> input2 = rowPagesBuilder(VARCHAR, BIGINT, DOUBLE, BOOLEAN).row("c", -1L, -0.1, true).row("d", 2L, 0.3, false).row("c", 4L, 0.2, true).pageBreak().row("d", 5L, 0.4, false).build();
    WindowOperatorFactory operatorFactory = createFactoryUnbounded(ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN), Ints.asList(0, 1, 2, 3), ROW_NUMBER, Ints.asList(0), Ints.asList(1), ImmutableList.copyOf(new SortOrder[] { SortOrder.ASC_NULLS_LAST }), false);
    DriverContext driverContext = createDriverContext(defaultMemoryLimit, TEST_SNAPSHOT_SESSION);
    WindowOperator windowOperator = (WindowOperator) operatorFactory.createOperator(driverContext);
    // Step1: add the first 2 pages
    for (Page page : input1) {
        windowOperator.addInput(page);
        windowOperator.getOutput();
    }
    // Step2: add a marker page to make 'capture1' happened
    MarkerPage marker = MarkerPage.snapshotPage(1);
    windowOperator.addInput(marker);
    windowOperator.getOutput();
    // Step3: add another 2 pages
    for (Page page : input2) {
        windowOperator.addInput(page);
        windowOperator.getOutput();
    }
    // Step4: assume the task is rescheduled due to failure and everything is re-constructed
    driverContext = createDriverContext(8, TEST_SNAPSHOT_SESSION);
    operatorFactory = createFactoryUnbounded(ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN), Ints.asList(0, 1, 2, 3), ROW_NUMBER, Ints.asList(0), Ints.asList(1), ImmutableList.copyOf(new SortOrder[] { SortOrder.ASC_NULLS_LAST }), false);
    windowOperator = (WindowOperator) operatorFactory.createOperator(driverContext);
    // Step5: restore to 'capture1'
    MarkerPage resumeMarker = MarkerPage.resumePage(1);
    windowOperator.addInput(resumeMarker);
    windowOperator.getOutput();
    // Step6: continue to add another 2 pages
    for (Page page : input2) {
        windowOperator.addInput(page);
        windowOperator.getOutput();
    }
    windowOperator.finish();
    // Compare the results
    MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, BIGINT, DOUBLE, BOOLEAN, BIGINT).row("a", 2L, 0.3, false, 1L).row("a", 4L, 0.2, true, 2L).row("a", 6L, 0.1, true, 3L).row("b", -1L, -0.1, true, 1L).row("b", 5L, 0.4, false, 2L).row("c", -1L, -0.1, true, 1L).row("c", 4L, 0.2, true, 2L).row("d", 2L, 0.3, false, 1L).row("d", 5L, 0.4, false, 2L).build();
    Page p = windowOperator.getOutput();
    while (p == null) {
        p = windowOperator.getOutput();
    }
    outputPages.add(p);
    MaterializedResult actual = toMaterializedResult(driverContext.getSession(), expected.getTypes(), outputPages.build());
    Assert.assertEquals(actual, expected);
}

Example 7 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class TestWindowOperator method testCaptureRestoreWithSpillToHdfsEnabled.

@Test
public void testCaptureRestoreWithSpillToHdfsEnabled() throws Exception {
    // Initialization
    Path spillPath = Paths.get("/tmp/hetu/snapshot/");
    HetuHdfsFileSystemClient fs = getLocalHdfs();
    when(fileSystemClientManager.getFileSystemClient(any(String.class), any(Path.class))).thenReturn(fs);
    GenericSpillerFactory genericSpillerFactory = createGenericSpillerFactory(spillPath, fileSystemClientManager, false, null);
    SnapshotConfig snapshotConfig = new SnapshotConfig();
    snapshotConfig.setSpillProfile("hdfs");
    snapshotConfig.setSpillToHdfs(true);
    snapshotUtils = new SnapshotUtils(fileSystemClientManager, snapshotConfig, new InMemoryNodeManager());
    snapshotUtils.initialize();
    ImmutableList.Builder<Page> outputPages = ImmutableList.builder();
    List<Page> input1 = rowPagesBuilder(VARCHAR, BIGINT, DOUBLE, BOOLEAN).row("b", -1L, -0.1, true).row("a", 2L, 0.3, false).row("a", 4L, 0.2, true).pageBreak().row("b", 5L, 0.4, false).row("a", 6L, 0.1, true).build();
    List<Page> input2 = rowPagesBuilder(VARCHAR, BIGINT, DOUBLE, BOOLEAN).row("c", -1L, -0.1, true).row("d", 2L, 0.3, false).row("c", 4L, 0.2, true).pageBreak().row("d", 5L, 0.4, false).build();
    WindowOperatorFactory operatorFactory = new WindowOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN), Ints.asList(0, 1, 2, 3), ROW_NUMBER, Ints.asList(0), ImmutableList.of(), Ints.asList(1), ImmutableList.copyOf(new SortOrder[] { SortOrder.ASC_NULLS_LAST }), 0, 10, new PagesIndex.TestingFactory(false), true, genericSpillerFactory, new OrderingCompiler());
    DriverContext driverContext = createDriverContext(defaultMemoryLimit, TEST_SNAPSHOT_SESSION);
    WindowOperator windowOperator = (WindowOperator) operatorFactory.createOperator(driverContext);
    // Step1: add the first 2 pages
    for (Page page : input1) {
        windowOperator.addInput(page);
        windowOperator.getOutput();
    }
    // Step2: spilling happened here
    getFutureValue(windowOperator.startMemoryRevoke());
    windowOperator.finishMemoryRevoke();
    // Step3: add a marker page to make 'capture1' happened
    MarkerPage marker = MarkerPage.snapshotPage(1);
    windowOperator.addInput(marker);
    windowOperator.getOutput();
    // Step4: add another 2 pages
    for (Page page : input2) {
        windowOperator.addInput(page);
        windowOperator.getOutput();
    }
    // Step5: assume the task is rescheduled due to failure and everything is re-constructed
    driverContext = createDriverContext(8, TEST_SNAPSHOT_SESSION);
    operatorFactory = new WindowOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN), Ints.asList(0, 1, 2, 3), ROW_NUMBER, Ints.asList(0), ImmutableList.of(), Ints.asList(1), ImmutableList.copyOf(new SortOrder[] { SortOrder.ASC_NULLS_LAST }), 0, 10, new PagesIndex.TestingFactory(false), true, genericSpillerFactory, new OrderingCompiler());
    windowOperator = (WindowOperator) operatorFactory.createOperator(driverContext);
    // Step6: restore to 'capture1', the spiller should contains the reference of the first 2 pages for now.
    MarkerPage resumeMarker = MarkerPage.resumePage(1);
    windowOperator.addInput(resumeMarker);
    windowOperator.getOutput();
    // Step7: continue to add another 2 pages
    for (Page page : input2) {
        windowOperator.addInput(page);
        windowOperator.getOutput();
    }
    windowOperator.finish();
    // Compare the results
    MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, BIGINT, DOUBLE, BOOLEAN, BIGINT).row("a", 2L, 0.3, false, 1L).row("a", 4L, 0.2, true, 2L).row("a", 6L, 0.1, true, 3L).row("b", -1L, -0.1, true, 1L).row("b", 5L, 0.4, false, 2L).row("c", -1L, -0.1, true, 1L).row("c", 4L, 0.2, true, 2L).row("d", 2L, 0.3, false, 1L).row("d", 5L, 0.4, false, 2L).build();
    Page p = windowOperator.getOutput();
    while (p == null) {
        p = windowOperator.getOutput();
    }
    outputPages.add(p);
    MaterializedResult actual = toMaterializedResult(driverContext.getSession(), expected.getTypes(), outputPages.build());
    Assert.assertEquals(actual, expected);
}

Example 8 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class TestValuesOperator method testSnapshot.

@Test
public void testSnapshot() {
    List<Page> pages = ImmutableList.of(new Page(1), MarkerPage.snapshotPage(1));
    ValuesOperator operator = new ValuesOperator(mockOperatorContext(true), pages, 0);
    Page page = operator.getOutput();
    assertNotNull(page);
    assertFalse(page instanceof MarkerPage);
    page = operator.getOutput();
    assertTrue(page instanceof MarkerPage);
    page = operator.getOutput();
    assertNull(page);
}

Example 9 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class PageSplitterUtil method splitPage.

private static List<Page> splitPage(Page page, long maxPageSizeInBytes, long previousPageSize) {
    checkArgument(page.getPositionCount() > 0, "page is empty");
    checkArgument(maxPageSizeInBytes > 0, "maxPageSizeInBytes must be > 0");
    // if the size of the page doesn't improve from the previous call we terminate the recursion.
    if (page instanceof MarkerPage || page.getSizeInBytes() == previousPageSize || page.getSizeInBytes() <= maxPageSizeInBytes || page.getPositionCount() == 1) {
        return ImmutableList.of(page);
    }
    ImmutableList.Builder<Page> outputPages = ImmutableList.builder();
    long previousSize = page.getSizeInBytes();
    int positionCount = page.getPositionCount();
    int half = positionCount / 2;
    Page leftHalf = page.getRegion(0, half);
    outputPages.addAll(splitPage(leftHalf, maxPageSizeInBytes, previousSize));
    Page rightHalf = page.getRegion(half, positionCount - half);
    outputPages.addAll(splitPage(rightHalf, maxPageSizeInBytes, previousSize));
    return outputPages.build();
}

Example 10 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class TestSingleInputSnapshotState method test2Resumes.

@Test
public void test2Resumes() throws Exception {
    processPage(regularPage);
    int saved1 = restorable.state;
    processPage(marker1);
    state.nextMarker();
    processPage(regularPage);
    int saved2 = restorable.state;
    processPage(marker2);
    state.nextMarker();
    processPage(regularPage);
    when(snapshotManager.loadState(snapshotId2)).thenReturn(Optional.of(saved2));
    state.processPage(resume2);
    processPage(regularPage);
    when(snapshotManager.loadState(snapshotId1)).thenReturn(Optional.of(saved1));
    state.processPage(resume1);
    Assert.assertEquals(restorable.state, saved1);
    state.processPage(regularPage);
    MarkerPage markerPage = state.nextMarker();
    Assert.assertEquals(markerPage.getSnapshotId(), 1);
    Assert.assertTrue(markerPage.isResuming());
    Assert.assertNull(state.nextMarker());
}