Examples with MarkerPage io.prestosql.spi.snapshot.MarkerPage used on opensource projects

Example 11 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class TestSingleInputSnapshotState method testResumeMarker.

@Test
public void testResumeMarker() throws Exception {
    processPage(regularPage);
    int saved = restorable.state;
    processPage(marker1);
    processPage(regularPage);
    when(snapshotManager.loadState(anyObject())).thenReturn(Optional.of(saved));
    boolean ret = state.processPage(resume1);
    Assert.assertTrue(ret);
    Assert.assertEquals(restorable.state, saved);
    MarkerPage markerPage = state.nextMarker();
    Assert.assertEquals(markerPage.getSnapshotId(), 1);
    Assert.assertTrue(markerPage.isResuming());
    Assert.assertNull(state.nextMarker());
}

Example 12 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class TestMarkerPage method testSerde.

@Test
public void testSerde() {
    MarkerPage page = new MarkerPage(100, true);
    SerializedPage spage = SerializedPage.forMarker(page);
    page = spage.toMarker();
    Assert.assertEquals(page.getSnapshotId(), 100);
    Assert.assertTrue(page.isResuming());
}

Example 13 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class TestMarkerPage method testStaticSerde.

@Test
public void testStaticSerde() {
    PagesSerde serde = TESTING_SERDE_FACTORY.createPagesSerde();
    MarkerPage marker1 = MarkerPage.snapshotPage(1);
    SerializedPage serializedPage = serde.serialize(marker1);
    Page deserializedPage = serde.deserialize(serializedPage);
    Assert.assertTrue(deserializedPage instanceof MarkerPage);
    marker1 = (MarkerPage) deserializedPage;
    Assert.assertEquals(marker1.getSnapshotId(), 1);
    Assert.assertFalse(marker1.isResuming());
    MarkerPage resume1 = MarkerPage.resumePage(1);
    serializedPage = serde.serialize(resume1);
    deserializedPage = serde.deserialize(serializedPage);
    Assert.assertTrue(deserializedPage instanceof MarkerPage);
    resume1 = (MarkerPage) deserializedPage;
    Assert.assertEquals(resume1.getSnapshotId(), 1);
    Assert.assertTrue(resume1.isResuming());
    Page regular = new Page(1);
    serializedPage = serde.serialize(regular);
    deserializedPage = serde.deserialize(serializedPage);
    Assert.assertFalse(deserializedPage instanceof MarkerPage);
}

Example 14 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class DeleteOperator method addInput.

@Override
public void addInput(Page page) {
    requireNonNull(page, "page is null");
    checkState(state == State.RUNNING, "Operator is %s", state);
    // TODO-cp-I38S9Q: Operator currently not supported for Snapshot
    if (page instanceof MarkerPage) {
        throw new UnsupportedOperationException("Operator doesn't support snapshotting.");
    }
    Block rowIds = page.getBlock(rowIdChannel);
    pageSource().deleteRows(rowIds);
    rowCount += rowIds.getPositionCount();
}

Example 15 with MarkerPage

use of io.prestosql.spi.snapshot.MarkerPage in project hetu-core by openlookeng.

the class SingleInputSnapshotState method processPage.

/**
 * Perform marker and snapshot related processing on an incoming input
 *
 * @param input the input, either a Page or a Split
 * @return true if the input has been processed, false otherwise (so operator needs to process it as a regular input)
 */
public boolean processPage(Page input) {
    if (!(input instanceof MarkerPage)) {
        return false;
    }
    MarkerPage marker = (MarkerPage) input;
    long snapshotId = marker.getSnapshotId();
    SnapshotStateId componentId = snapshotStateIdGenerator.apply(snapshotId);
    if (marker.isResuming()) {
        try {
            Optional<Object> state;
            if (restorable.supportsConsolidatedWrites()) {
                state = snapshotManager.loadConsolidatedState(componentId);
            } else {
                state = snapshotManager.loadState(componentId);
            }
            if (!state.isPresent()) {
                snapshotManager.failedToRestore(componentId, true);
                LOG.warn("Can't locate saved state for snapshot %d, component %s", snapshotId, restorableId);
            } else if (state.get() == TaskSnapshotManager.NO_STATE) {
                snapshotManager.failedToRestore(componentId, true);
                LOG.error("BUG! State of component %s has never been stored successfully before snapshot %d", restorableId, snapshotId);
            } else {
                Stopwatch timer = Stopwatch.createStarted();
                restorable.restore(state.get(), pagesSerde);
                timer.stop();
                boolean successful = true;
                if (restorable instanceof Spillable && ((Spillable) restorable).isSpilled()) {
                    Boolean result = loadSpilledFiles(snapshotId, (Spillable) restorable);
                    if (result == null) {
                        snapshotManager.failedToRestore(componentId, true);
                        LOG.error("BUG! Spilled file of component %s has never been stored successfully before snapshot %d", restorableId, snapshotId);
                        successful = false;
                    } else if (!result) {
                        snapshotManager.failedToRestore(componentId, true);
                        LOG.warn("Can't locate spilled file for snapshot %d, component %s", snapshotId, restorableId);
                        successful = false;
                    }
                }
                if (successful) {
                    LOG.debug("Successfully restored state to snapshot %d for %s", snapshotId, restorableId);
                    snapshotManager.succeededToRestore(componentId, timer.elapsed(TimeUnit.MILLISECONDS));
                }
            }
            // Previous pending snapshots no longer need to be carried out
            markers.clear();
        } catch (Exception e) {
            LOG.warn(e, "Failed to restore snapshot state for %s: %s", componentId, e.getMessage());
            snapshotManager.failedToRestore(componentId, false);
        }
    } else {
        captureState(snapshotId, true);
    }
    markers.add(marker);
    return true;
}