Search in sources :

Example 1 with OperatorSnapshotFinalizer

use of org.apache.flink.streaming.api.operators.OperatorSnapshotFinalizer in project flink by apache.

the class AbstractStreamOperatorTestHarness method snapshotWithLocalState.

/**
 * Calls {@link StreamOperator#snapshotState(long, long, CheckpointOptions,
 * org.apache.flink.runtime.state.CheckpointStreamFactory)}.
 */
public OperatorSnapshotFinalizer snapshotWithLocalState(long checkpointId, long timestamp, SnapshotType checkpointType) throws Exception {
    CheckpointStorageLocationReference locationReference = CheckpointStorageLocationReference.getDefault();
    OperatorSnapshotFutures operatorStateResult = operator.snapshotState(checkpointId, timestamp, new CheckpointOptions(checkpointType, locationReference), checkpointStorageAccess.resolveCheckpointStorageLocation(checkpointId, locationReference));
    return new OperatorSnapshotFinalizer(operatorStateResult);
}
Also used : OperatorSnapshotFutures(org.apache.flink.streaming.api.operators.OperatorSnapshotFutures) CheckpointOptions(org.apache.flink.runtime.checkpoint.CheckpointOptions) CheckpointStorageLocationReference(org.apache.flink.runtime.state.CheckpointStorageLocationReference) OperatorSnapshotFinalizer(org.apache.flink.streaming.api.operators.OperatorSnapshotFinalizer)

Example 2 with OperatorSnapshotFinalizer

use of org.apache.flink.streaming.api.operators.OperatorSnapshotFinalizer in project flink by apache.

the class SnapshotUtils method snapshot.

public static <OUT, OP extends StreamOperator<OUT>> TaggedOperatorSubtaskState snapshot(OP operator, int index, long timestamp, boolean isExactlyOnceMode, boolean isUnalignedCheckpoint, Configuration configuration, Path savepointPath) throws Exception {
    CheckpointOptions options = CheckpointOptions.forConfig(SavepointType.savepoint(SavepointFormatType.CANONICAL), AbstractFsCheckpointStorageAccess.encodePathAsReference(savepointPath), isExactlyOnceMode, isUnalignedCheckpoint, CheckpointOptions.NO_ALIGNED_CHECKPOINT_TIME_OUT);
    operator.prepareSnapshotPreBarrier(CHECKPOINT_ID);
    CheckpointStreamFactory storage = createStreamFactory(configuration, options);
    OperatorSnapshotFutures snapshotInProgress = operator.snapshotState(CHECKPOINT_ID, timestamp, options, storage);
    OperatorSubtaskState state = new OperatorSnapshotFinalizer(snapshotInProgress).getJobManagerOwnedState();
    operator.notifyCheckpointComplete(CHECKPOINT_ID);
    return new TaggedOperatorSubtaskState(index, state);
}
Also used : OperatorSnapshotFutures(org.apache.flink.streaming.api.operators.OperatorSnapshotFutures) CheckpointStreamFactory(org.apache.flink.runtime.state.CheckpointStreamFactory) CheckpointOptions(org.apache.flink.runtime.checkpoint.CheckpointOptions) OperatorSubtaskState(org.apache.flink.runtime.checkpoint.OperatorSubtaskState) OperatorSnapshotFinalizer(org.apache.flink.streaming.api.operators.OperatorSnapshotFinalizer)

Example 3 with OperatorSnapshotFinalizer

use of org.apache.flink.streaming.api.operators.OperatorSnapshotFinalizer in project flink by apache.

the class AsyncCheckpointRunnable method finalizeNonFinishedSnapshots.

private SnapshotsFinalizeResult finalizeNonFinishedSnapshots() throws Exception {
    TaskStateSnapshot jobManagerTaskOperatorSubtaskStates = new TaskStateSnapshot(operatorSnapshotsInProgress.size(), isTaskFinished);
    TaskStateSnapshot localTaskOperatorSubtaskStates = new TaskStateSnapshot(operatorSnapshotsInProgress.size(), isTaskFinished);
    long bytesPersistedDuringAlignment = 0;
    for (Map.Entry<OperatorID, OperatorSnapshotFutures> entry : operatorSnapshotsInProgress.entrySet()) {
        OperatorID operatorID = entry.getKey();
        OperatorSnapshotFutures snapshotInProgress = entry.getValue();
        // finalize the async part of all by executing all snapshot runnables
        OperatorSnapshotFinalizer finalizedSnapshots = new OperatorSnapshotFinalizer(snapshotInProgress);
        jobManagerTaskOperatorSubtaskStates.putSubtaskStateByOperatorID(operatorID, finalizedSnapshots.getJobManagerOwnedState());
        localTaskOperatorSubtaskStates.putSubtaskStateByOperatorID(operatorID, finalizedSnapshots.getTaskLocalState());
        bytesPersistedDuringAlignment += finalizedSnapshots.getJobManagerOwnedState().getResultSubpartitionState().getStateSize();
        bytesPersistedDuringAlignment += finalizedSnapshots.getJobManagerOwnedState().getInputChannelState().getStateSize();
    }
    return new SnapshotsFinalizeResult(jobManagerTaskOperatorSubtaskStates, localTaskOperatorSubtaskStates, bytesPersistedDuringAlignment);
}
Also used : OperatorSnapshotFutures(org.apache.flink.streaming.api.operators.OperatorSnapshotFutures) TaskStateSnapshot(org.apache.flink.runtime.checkpoint.TaskStateSnapshot) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) Map(java.util.Map) OperatorSnapshotFinalizer(org.apache.flink.streaming.api.operators.OperatorSnapshotFinalizer)

Example 4 with OperatorSnapshotFinalizer

use of org.apache.flink.streaming.api.operators.OperatorSnapshotFinalizer in project flink by apache.

the class StreamOperatorSnapshotRestoreTest method testOperatorStatesSnapshotRestoreInternal.

private void testOperatorStatesSnapshotRestoreInternal(final int mode) throws Exception {
    // -------------------------------------------------------------------------- snapshot
    StateBackend stateBackend;
    FsStateBackend fsstateBackend = createStateBackendInternal();
    switch(stateBackendEnum) {
        case FILE:
            stateBackend = fsstateBackend;
            break;
        case ROCKSDB_FULLY_ASYNC:
            stateBackend = new RocksDBStateBackend(fsstateBackend, TernaryBoolean.FALSE);
            break;
        case ROCKSDB_INCREMENTAL:
            stateBackend = new RocksDBStateBackend(fsstateBackend, TernaryBoolean.TRUE);
            break;
        default:
            throw new IllegalStateException(String.format("Do not support statebackend type %s", stateBackendEnum));
    }
    TestOneInputStreamOperator op = new TestOneInputStreamOperator(false);
    JobID jobID = new JobID();
    JobVertexID jobVertexID = new JobVertexID();
    int subtaskIdx = 0;
    LocalRecoveryDirectoryProvider directoryProvider = mode == ONLY_JM_RECOVERY ? null : new LocalRecoveryDirectoryProviderImpl(temporaryFolder.newFolder(), jobID, jobVertexID, subtaskIdx);
    LocalRecoveryConfig localRecoveryConfig = new LocalRecoveryConfig(directoryProvider);
    MockEnvironment mockEnvironment = new MockEnvironmentBuilder().setJobID(jobID).setJobVertexID(jobVertexID).setTaskName("test").setManagedMemorySize(1024L * 1024L).setInputSplitProvider(new MockInputSplitProvider()).setBufferSize(1024 * 1024).setTaskStateManager(new TestTaskStateManager(localRecoveryConfig)).setMaxParallelism(MAX_PARALLELISM).setSubtaskIndex(subtaskIdx).setUserCodeClassLoader(getClass().getClassLoader()).build();
    KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Integer> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(op, (KeySelector<Integer, Integer>) value -> value, TypeInformation.of(Integer.class), mockEnvironment);
    testHarness.setStateBackend(stateBackend);
    testHarness.open();
    for (int i = 0; i < 10; ++i) {
        testHarness.processElement(new StreamRecord<>(i));
    }
    OperatorSnapshotFinalizer snapshotWithLocalState = testHarness.snapshotWithLocalState(1L, 1L);
    testHarness.close();
    // -------------------------------------------------------------------------- restore
    op = new TestOneInputStreamOperator(true);
    testHarness = new KeyedOneInputStreamOperatorTestHarness<>(op, (KeySelector<Integer, Integer>) value -> value, TypeInformation.of(Integer.class), MAX_PARALLELISM, 1, /* num subtasks */
    0);
    testHarness.setTimeServiceManagerProvider(new InternalTimeServiceManager.Provider() {

        @Override
        public <K> InternalTimeServiceManager<K> create(CheckpointableKeyedStateBackend<K> keyedStatedBackend, ClassLoader userClassloader, KeyContext keyContext, ProcessingTimeService processingTimeService, Iterable<KeyGroupStatePartitionStreamProvider> rawKeyedStates) throws IOException {
            return null;
        }
    });
    testHarness.setStateBackend(stateBackend);
    OperatorSubtaskState jobManagerOwnedState = snapshotWithLocalState.getJobManagerOwnedState();
    OperatorSubtaskState taskLocalState = snapshotWithLocalState.getTaskLocalState();
    // We check if local state was created when we enabled local recovery
    Assert.assertTrue(mode > ONLY_JM_RECOVERY == (taskLocalState != null && taskLocalState.hasState()));
    if (mode == TM_REMOVE_JM_RECOVERY) {
        jobManagerOwnedState.getManagedKeyedState().discardState();
    } else if (mode == JM_REMOVE_TM_RECOVERY) {
        taskLocalState.getManagedKeyedState().discardState();
    }
    testHarness.initializeState(jobManagerOwnedState, taskLocalState);
    testHarness.open();
    for (int i = 0; i < 10; ++i) {
        testHarness.processElement(new StreamRecord<>(i));
    }
    testHarness.close();
}
Also used : InternalTimeServiceManager(org.apache.flink.streaming.api.operators.InternalTimeServiceManager) Arrays(java.util.Arrays) OperatorSnapshotFinalizer(org.apache.flink.streaming.api.operators.OperatorSnapshotFinalizer) LocalRecoveryConfig(org.apache.flink.runtime.state.LocalRecoveryConfig) KeyedOneInputStreamOperatorTestHarness(org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness) DataOutputViewStreamWrapper(org.apache.flink.core.memory.DataOutputViewStreamWrapper) ListState(org.apache.flink.api.common.state.ListState) IntSerializer(org.apache.flink.api.common.typeutils.base.IntSerializer) StateBackend(org.apache.flink.runtime.state.StateBackend) TestTaskStateManager(org.apache.flink.runtime.state.TestTaskStateManager) TestLogger(org.apache.flink.util.TestLogger) ListStateDescriptor(org.apache.flink.api.common.state.ListStateDescriptor) TypeInformation(org.apache.flink.api.common.typeinfo.TypeInformation) Parameterized(org.junit.runners.Parameterized) LocalRecoveryDirectoryProviderImpl(org.apache.flink.runtime.state.LocalRecoveryDirectoryProviderImpl) DataInputViewStreamWrapper(org.apache.flink.core.memory.DataInputViewStreamWrapper) StateSnapshotContext(org.apache.flink.runtime.state.StateSnapshotContext) AfterClass(org.junit.AfterClass) KeySelector(org.apache.flink.api.java.functions.KeySelector) KeyContext(org.apache.flink.streaming.api.operators.KeyContext) MockInputSplitProvider(org.apache.flink.runtime.operators.testutils.MockInputSplitProvider) Collection(java.util.Collection) FsStateBackend(org.apache.flink.runtime.state.filesystem.FsStateBackend) OperatorSubtaskState(org.apache.flink.runtime.checkpoint.OperatorSubtaskState) TernaryBoolean(org.apache.flink.util.TernaryBoolean) ValueState(org.apache.flink.api.common.state.ValueState) OperatorStateCheckpointOutputStream(org.apache.flink.runtime.state.OperatorStateCheckpointOutputStream) KeyGroupStatePartitionStreamProvider(org.apache.flink.runtime.state.KeyGroupStatePartitionStreamProvider) OneInputStreamOperator(org.apache.flink.streaming.api.operators.OneInputStreamOperator) LocalRecoveryDirectoryProvider(org.apache.flink.runtime.state.LocalRecoveryDirectoryProvider) StatePartitionStreamProvider(org.apache.flink.runtime.state.StatePartitionStreamProvider) BeforeClass(org.junit.BeforeClass) KeyedStateCheckpointOutputStream(org.apache.flink.runtime.state.KeyedStateCheckpointOutputStream) RunWith(org.junit.runner.RunWith) Watermark(org.apache.flink.streaming.api.watermark.Watermark) DataOutputView(org.apache.flink.core.memory.DataOutputView) JobVertexID(org.apache.flink.runtime.jobgraph.JobVertexID) StreamRecord(org.apache.flink.streaming.runtime.streamrecord.StreamRecord) DataInputView(org.apache.flink.core.memory.DataInputView) MockEnvironment(org.apache.flink.runtime.operators.testutils.MockEnvironment) CheckpointableKeyedStateBackend(org.apache.flink.runtime.state.CheckpointableKeyedStateBackend) ValueStateDescriptor(org.apache.flink.api.common.state.ValueStateDescriptor) ProcessingTimeService(org.apache.flink.streaming.runtime.tasks.ProcessingTimeService) Test(org.junit.Test) IOException(java.io.IOException) MockEnvironmentBuilder(org.apache.flink.runtime.operators.testutils.MockEnvironmentBuilder) File(java.io.File) AbstractStreamOperator(org.apache.flink.streaming.api.operators.AbstractStreamOperator) RocksDBStateBackend(org.apache.flink.contrib.streaming.state.RocksDBStateBackend) StreamOperator(org.apache.flink.streaming.api.operators.StreamOperator) JobID(org.apache.flink.api.common.JobID) BitSet(java.util.BitSet) Assert(org.junit.Assert) TemporaryFolder(org.junit.rules.TemporaryFolder) StateInitializationContext(org.apache.flink.runtime.state.StateInitializationContext) InputStream(java.io.InputStream) RocksDBStateBackend(org.apache.flink.contrib.streaming.state.RocksDBStateBackend) MockEnvironmentBuilder(org.apache.flink.runtime.operators.testutils.MockEnvironmentBuilder) JobVertexID(org.apache.flink.runtime.jobgraph.JobVertexID) LocalRecoveryDirectoryProvider(org.apache.flink.runtime.state.LocalRecoveryDirectoryProvider) LocalRecoveryDirectoryProviderImpl(org.apache.flink.runtime.state.LocalRecoveryDirectoryProviderImpl) KeySelector(org.apache.flink.api.java.functions.KeySelector) StateBackend(org.apache.flink.runtime.state.StateBackend) FsStateBackend(org.apache.flink.runtime.state.filesystem.FsStateBackend) CheckpointableKeyedStateBackend(org.apache.flink.runtime.state.CheckpointableKeyedStateBackend) RocksDBStateBackend(org.apache.flink.contrib.streaming.state.RocksDBStateBackend) KeyedOneInputStreamOperatorTestHarness(org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness) OperatorSubtaskState(org.apache.flink.runtime.checkpoint.OperatorSubtaskState) KeyGroupStatePartitionStreamProvider(org.apache.flink.runtime.state.KeyGroupStatePartitionStreamProvider) KeyContext(org.apache.flink.streaming.api.operators.KeyContext) MockEnvironment(org.apache.flink.runtime.operators.testutils.MockEnvironment) ProcessingTimeService(org.apache.flink.streaming.runtime.tasks.ProcessingTimeService) MockInputSplitProvider(org.apache.flink.runtime.operators.testutils.MockInputSplitProvider) InternalTimeServiceManager(org.apache.flink.streaming.api.operators.InternalTimeServiceManager) LocalRecoveryConfig(org.apache.flink.runtime.state.LocalRecoveryConfig) IOException(java.io.IOException) OperatorSnapshotFinalizer(org.apache.flink.streaming.api.operators.OperatorSnapshotFinalizer) TestTaskStateManager(org.apache.flink.runtime.state.TestTaskStateManager) FsStateBackend(org.apache.flink.runtime.state.filesystem.FsStateBackend) JobID(org.apache.flink.api.common.JobID)

Aggregations

OperatorSnapshotFinalizer (org.apache.flink.streaming.api.operators.OperatorSnapshotFinalizer)4 OperatorSnapshotFutures (org.apache.flink.streaming.api.operators.OperatorSnapshotFutures)3 CheckpointOptions (org.apache.flink.runtime.checkpoint.CheckpointOptions)2 OperatorSubtaskState (org.apache.flink.runtime.checkpoint.OperatorSubtaskState)2 File (java.io.File)1 IOException (java.io.IOException)1 InputStream (java.io.InputStream)1 Arrays (java.util.Arrays)1 BitSet (java.util.BitSet)1 Collection (java.util.Collection)1 Map (java.util.Map)1 JobID (org.apache.flink.api.common.JobID)1 ListState (org.apache.flink.api.common.state.ListState)1 ListStateDescriptor (org.apache.flink.api.common.state.ListStateDescriptor)1 ValueState (org.apache.flink.api.common.state.ValueState)1 ValueStateDescriptor (org.apache.flink.api.common.state.ValueStateDescriptor)1 TypeInformation (org.apache.flink.api.common.typeinfo.TypeInformation)1 IntSerializer (org.apache.flink.api.common.typeutils.base.IntSerializer)1 KeySelector (org.apache.flink.api.java.functions.KeySelector)1 RocksDBStateBackend (org.apache.flink.contrib.streaming.state.RocksDBStateBackend)1