Examples with OperatorSnapshotFutures org.apache.flink.streaming.api.operators.OperatorSnapshotFutures used on opensource projects

Example 11 with OperatorSnapshotFutures

use of org.apache.flink.streaming.api.operators.OperatorSnapshotFutures in project flink by apache.

the class SubtaskCheckpointCoordinatorTest method testNotifyCheckpointAbortedDuringAsyncPhase.

@Test
public void testNotifyCheckpointAbortedDuringAsyncPhase() throws Exception {
    MockEnvironment mockEnvironment = MockEnvironment.builder().build();
    try (SubtaskCheckpointCoordinatorImpl subtaskCheckpointCoordinator = (SubtaskCheckpointCoordinatorImpl) new MockSubtaskCheckpointCoordinatorBuilder().setEnvironment(mockEnvironment).setExecutor(Executors.newSingleThreadExecutor()).setUnalignedCheckpointEnabled(true).build()) {
        final BlockingRunnableFuture rawKeyedStateHandleFuture = new BlockingRunnableFuture();
        OperatorSnapshotFutures operatorSnapshotResult = new OperatorSnapshotFutures(DoneFuture.of(SnapshotResult.empty()), rawKeyedStateHandleFuture, DoneFuture.of(SnapshotResult.empty()), DoneFuture.of(SnapshotResult.empty()), DoneFuture.of(SnapshotResult.empty()), DoneFuture.of(SnapshotResult.empty()));
        final OperatorChain<String, AbstractStreamOperator<String>> operatorChain = operatorChain(new CheckpointOperator(operatorSnapshotResult));
        long checkpointId = 42L;
        subtaskCheckpointCoordinator.getChannelStateWriter().start(checkpointId, CheckpointOptions.forCheckpointWithDefaultLocation());
        subtaskCheckpointCoordinator.checkpointState(new CheckpointMetaData(checkpointId, System.currentTimeMillis()), CheckpointOptions.forCheckpointWithDefaultLocation(), new CheckpointMetricsBuilder(), operatorChain, false, () -> false);
        rawKeyedStateHandleFuture.awaitRun();
        assertEquals(1, subtaskCheckpointCoordinator.getAsyncCheckpointRunnableSize());
        assertFalse(rawKeyedStateHandleFuture.isCancelled());
        subtaskCheckpointCoordinator.notifyCheckpointAborted(checkpointId, operatorChain, () -> true);
        assertTrue(rawKeyedStateHandleFuture.isCancelled());
        assertEquals(0, subtaskCheckpointCoordinator.getAsyncCheckpointRunnableSize());
    }
}

Example 12 with OperatorSnapshotFutures

use of org.apache.flink.streaming.api.operators.OperatorSnapshotFutures in project flink by apache.

the class SnapshotUtils method snapshot.

public static <OUT, OP extends StreamOperator<OUT>> TaggedOperatorSubtaskState snapshot(OP operator, int index, long timestamp, boolean isExactlyOnceMode, boolean isUnalignedCheckpoint, Configuration configuration, Path savepointPath) throws Exception {
    CheckpointOptions options = CheckpointOptions.forConfig(SavepointType.savepoint(SavepointFormatType.CANONICAL), AbstractFsCheckpointStorageAccess.encodePathAsReference(savepointPath), isExactlyOnceMode, isUnalignedCheckpoint, CheckpointOptions.NO_ALIGNED_CHECKPOINT_TIME_OUT);
    operator.prepareSnapshotPreBarrier(CHECKPOINT_ID);
    CheckpointStreamFactory storage = createStreamFactory(configuration, options);
    OperatorSnapshotFutures snapshotInProgress = operator.snapshotState(CHECKPOINT_ID, timestamp, options, storage);
    OperatorSubtaskState state = new OperatorSnapshotFinalizer(snapshotInProgress).getJobManagerOwnedState();
    operator.notifyCheckpointComplete(CHECKPOINT_ID);
    return new TaggedOperatorSubtaskState(index, state);
}

Example 13 with OperatorSnapshotFutures

use of org.apache.flink.streaming.api.operators.OperatorSnapshotFutures in project flink by apache.

the class AsyncCheckpointRunnable method cleanup.

/**
 * @return discarded full/incremental size (if available).
 */
private Tuple2<Long, Long> cleanup() throws Exception {
    LOG.debug("Cleanup AsyncCheckpointRunnable for checkpoint {} of {}.", checkpointMetaData.getCheckpointId(), taskName);
    Exception exception = null;
    // clean up ongoing operator snapshot results and non partitioned state handles
    long stateSize = 0, checkpointedSize = 0;
    for (OperatorSnapshotFutures operatorSnapshotResult : operatorSnapshotsInProgress.values()) {
        if (operatorSnapshotResult != null) {
            try {
                Tuple2<Long, Long> tuple2 = operatorSnapshotResult.cancel();
                stateSize += tuple2.f0;
                checkpointedSize += tuple2.f1;
            } catch (Exception cancelException) {
                exception = ExceptionUtils.firstOrSuppressed(cancelException, exception);
            }
        }
    }
    if (null != exception) {
        throw exception;
    }
    return Tuple2.of(stateSize, checkpointedSize);
}

Example 14 with OperatorSnapshotFutures

use of org.apache.flink.streaming.api.operators.OperatorSnapshotFutures in project flink by apache.

the class AsyncCheckpointRunnable method finalizeNonFinishedSnapshots.

private SnapshotsFinalizeResult finalizeNonFinishedSnapshots() throws Exception {
    TaskStateSnapshot jobManagerTaskOperatorSubtaskStates = new TaskStateSnapshot(operatorSnapshotsInProgress.size(), isTaskFinished);
    TaskStateSnapshot localTaskOperatorSubtaskStates = new TaskStateSnapshot(operatorSnapshotsInProgress.size(), isTaskFinished);
    long bytesPersistedDuringAlignment = 0;
    for (Map.Entry<OperatorID, OperatorSnapshotFutures> entry : operatorSnapshotsInProgress.entrySet()) {
        OperatorID operatorID = entry.getKey();
        OperatorSnapshotFutures snapshotInProgress = entry.getValue();
        // finalize the async part of all by executing all snapshot runnables
        OperatorSnapshotFinalizer finalizedSnapshots = new OperatorSnapshotFinalizer(snapshotInProgress);
        jobManagerTaskOperatorSubtaskStates.putSubtaskStateByOperatorID(operatorID, finalizedSnapshots.getJobManagerOwnedState());
        localTaskOperatorSubtaskStates.putSubtaskStateByOperatorID(operatorID, finalizedSnapshots.getTaskLocalState());
        bytesPersistedDuringAlignment += finalizedSnapshots.getJobManagerOwnedState().getResultSubpartitionState().getStateSize();
        bytesPersistedDuringAlignment += finalizedSnapshots.getJobManagerOwnedState().getInputChannelState().getStateSize();
    }
    return new SnapshotsFinalizeResult(jobManagerTaskOperatorSubtaskStates, localTaskOperatorSubtaskStates, bytesPersistedDuringAlignment);
}

Example 15 with OperatorSnapshotFutures

use of org.apache.flink.streaming.api.operators.OperatorSnapshotFutures in project flink by apache.

the class FinishedOperatorChain method snapshotState.

@Override
public void snapshotState(Map<OperatorID, OperatorSnapshotFutures> operatorSnapshotsInProgress, CheckpointMetaData checkpointMetaData, CheckpointOptions checkpointOptions, Supplier<Boolean> isRunning, ChannelStateWriter.ChannelStateWriteResult channelStateWriteResult, CheckpointStreamFactory storage) throws Exception {
    for (StreamOperatorWrapper<?, ?> operatorWrapper : getAllOperators(true)) {
        StreamOperator<?> operator = operatorWrapper.getStreamOperator();
        if (operator == getMainOperator() || operator == getTailOperator()) {
            OperatorSnapshotFutures snapshotInProgress = new OperatorSnapshotFutures();
            snapshotChannelStates(operator, channelStateWriteResult, snapshotInProgress);
            operatorSnapshotsInProgress.put(operatorWrapper.getStreamOperator().getOperatorID(), snapshotInProgress);
        }
    }
}