use of org.apache.flink.streaming.api.operators.OperatorSnapshotFinalizer in project flink by apache.
the class AbstractStreamOperatorTestHarness method snapshotWithLocalState.
/**
* Calls {@link StreamOperator#snapshotState(long, long, CheckpointOptions,
* org.apache.flink.runtime.state.CheckpointStreamFactory)}.
*/
public OperatorSnapshotFinalizer snapshotWithLocalState(long checkpointId, long timestamp, SnapshotType checkpointType) throws Exception {
CheckpointStorageLocationReference locationReference = CheckpointStorageLocationReference.getDefault();
OperatorSnapshotFutures operatorStateResult = operator.snapshotState(checkpointId, timestamp, new CheckpointOptions(checkpointType, locationReference), checkpointStorageAccess.resolveCheckpointStorageLocation(checkpointId, locationReference));
return new OperatorSnapshotFinalizer(operatorStateResult);
}
use of org.apache.flink.streaming.api.operators.OperatorSnapshotFinalizer in project flink by apache.
the class SnapshotUtils method snapshot.
public static <OUT, OP extends StreamOperator<OUT>> TaggedOperatorSubtaskState snapshot(OP operator, int index, long timestamp, boolean isExactlyOnceMode, boolean isUnalignedCheckpoint, Configuration configuration, Path savepointPath) throws Exception {
CheckpointOptions options = CheckpointOptions.forConfig(SavepointType.savepoint(SavepointFormatType.CANONICAL), AbstractFsCheckpointStorageAccess.encodePathAsReference(savepointPath), isExactlyOnceMode, isUnalignedCheckpoint, CheckpointOptions.NO_ALIGNED_CHECKPOINT_TIME_OUT);
operator.prepareSnapshotPreBarrier(CHECKPOINT_ID);
CheckpointStreamFactory storage = createStreamFactory(configuration, options);
OperatorSnapshotFutures snapshotInProgress = operator.snapshotState(CHECKPOINT_ID, timestamp, options, storage);
OperatorSubtaskState state = new OperatorSnapshotFinalizer(snapshotInProgress).getJobManagerOwnedState();
operator.notifyCheckpointComplete(CHECKPOINT_ID);
return new TaggedOperatorSubtaskState(index, state);
}
use of org.apache.flink.streaming.api.operators.OperatorSnapshotFinalizer in project flink by apache.
the class AsyncCheckpointRunnable method finalizeNonFinishedSnapshots.
private SnapshotsFinalizeResult finalizeNonFinishedSnapshots() throws Exception {
TaskStateSnapshot jobManagerTaskOperatorSubtaskStates = new TaskStateSnapshot(operatorSnapshotsInProgress.size(), isTaskFinished);
TaskStateSnapshot localTaskOperatorSubtaskStates = new TaskStateSnapshot(operatorSnapshotsInProgress.size(), isTaskFinished);
long bytesPersistedDuringAlignment = 0;
for (Map.Entry<OperatorID, OperatorSnapshotFutures> entry : operatorSnapshotsInProgress.entrySet()) {
OperatorID operatorID = entry.getKey();
OperatorSnapshotFutures snapshotInProgress = entry.getValue();
// finalize the async part of all by executing all snapshot runnables
OperatorSnapshotFinalizer finalizedSnapshots = new OperatorSnapshotFinalizer(snapshotInProgress);
jobManagerTaskOperatorSubtaskStates.putSubtaskStateByOperatorID(operatorID, finalizedSnapshots.getJobManagerOwnedState());
localTaskOperatorSubtaskStates.putSubtaskStateByOperatorID(operatorID, finalizedSnapshots.getTaskLocalState());
bytesPersistedDuringAlignment += finalizedSnapshots.getJobManagerOwnedState().getResultSubpartitionState().getStateSize();
bytesPersistedDuringAlignment += finalizedSnapshots.getJobManagerOwnedState().getInputChannelState().getStateSize();
}
return new SnapshotsFinalizeResult(jobManagerTaskOperatorSubtaskStates, localTaskOperatorSubtaskStates, bytesPersistedDuringAlignment);
}
use of org.apache.flink.streaming.api.operators.OperatorSnapshotFinalizer in project flink by apache.
the class StreamOperatorSnapshotRestoreTest method testOperatorStatesSnapshotRestoreInternal.
private void testOperatorStatesSnapshotRestoreInternal(final int mode) throws Exception {
// -------------------------------------------------------------------------- snapshot
StateBackend stateBackend;
FsStateBackend fsstateBackend = createStateBackendInternal();
switch(stateBackendEnum) {
case FILE:
stateBackend = fsstateBackend;
break;
case ROCKSDB_FULLY_ASYNC:
stateBackend = new RocksDBStateBackend(fsstateBackend, TernaryBoolean.FALSE);
break;
case ROCKSDB_INCREMENTAL:
stateBackend = new RocksDBStateBackend(fsstateBackend, TernaryBoolean.TRUE);
break;
default:
throw new IllegalStateException(String.format("Do not support statebackend type %s", stateBackendEnum));
}
TestOneInputStreamOperator op = new TestOneInputStreamOperator(false);
JobID jobID = new JobID();
JobVertexID jobVertexID = new JobVertexID();
int subtaskIdx = 0;
LocalRecoveryDirectoryProvider directoryProvider = mode == ONLY_JM_RECOVERY ? null : new LocalRecoveryDirectoryProviderImpl(temporaryFolder.newFolder(), jobID, jobVertexID, subtaskIdx);
LocalRecoveryConfig localRecoveryConfig = new LocalRecoveryConfig(directoryProvider);
MockEnvironment mockEnvironment = new MockEnvironmentBuilder().setJobID(jobID).setJobVertexID(jobVertexID).setTaskName("test").setManagedMemorySize(1024L * 1024L).setInputSplitProvider(new MockInputSplitProvider()).setBufferSize(1024 * 1024).setTaskStateManager(new TestTaskStateManager(localRecoveryConfig)).setMaxParallelism(MAX_PARALLELISM).setSubtaskIndex(subtaskIdx).setUserCodeClassLoader(getClass().getClassLoader()).build();
KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Integer> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(op, (KeySelector<Integer, Integer>) value -> value, TypeInformation.of(Integer.class), mockEnvironment);
testHarness.setStateBackend(stateBackend);
testHarness.open();
for (int i = 0; i < 10; ++i) {
testHarness.processElement(new StreamRecord<>(i));
}
OperatorSnapshotFinalizer snapshotWithLocalState = testHarness.snapshotWithLocalState(1L, 1L);
testHarness.close();
// -------------------------------------------------------------------------- restore
op = new TestOneInputStreamOperator(true);
testHarness = new KeyedOneInputStreamOperatorTestHarness<>(op, (KeySelector<Integer, Integer>) value -> value, TypeInformation.of(Integer.class), MAX_PARALLELISM, 1, /* num subtasks */
0);
testHarness.setTimeServiceManagerProvider(new InternalTimeServiceManager.Provider() {
@Override
public <K> InternalTimeServiceManager<K> create(CheckpointableKeyedStateBackend<K> keyedStatedBackend, ClassLoader userClassloader, KeyContext keyContext, ProcessingTimeService processingTimeService, Iterable<KeyGroupStatePartitionStreamProvider> rawKeyedStates) throws IOException {
return null;
}
});
testHarness.setStateBackend(stateBackend);
OperatorSubtaskState jobManagerOwnedState = snapshotWithLocalState.getJobManagerOwnedState();
OperatorSubtaskState taskLocalState = snapshotWithLocalState.getTaskLocalState();
// We check if local state was created when we enabled local recovery
Assert.assertTrue(mode > ONLY_JM_RECOVERY == (taskLocalState != null && taskLocalState.hasState()));
if (mode == TM_REMOVE_JM_RECOVERY) {
jobManagerOwnedState.getManagedKeyedState().discardState();
} else if (mode == JM_REMOVE_TM_RECOVERY) {
taskLocalState.getManagedKeyedState().discardState();
}
testHarness.initializeState(jobManagerOwnedState, taskLocalState);
testHarness.open();
for (int i = 0; i < 10; ++i) {
testHarness.processElement(new StreamRecord<>(i));
}
testHarness.close();
}
Aggregations