Examples with IncrementalRemoteKeyedStateHandle org.apache.flink.runtime.state.IncrementalRemoteKeyedStateHandle used on opensource projects

Example 6 with IncrementalRemoteKeyedStateHandle

use of org.apache.flink.runtime.state.IncrementalRemoteKeyedStateHandle in project flink by apache.

the class RocksDBStateDownloaderTest method testMultiThreadRestoreThreadPoolExceptionRethrow.

/**
 * Test that the exception arose in the thread pool will rethrow to the main thread.
 */
@Test
public void testMultiThreadRestoreThreadPoolExceptionRethrow() {
    SpecifiedException expectedException = new SpecifiedException("throw exception while multi thread restore.");
    StreamStateHandle stateHandle = new ThrowingStateHandle(expectedException);
    Map<StateHandleID, StreamStateHandle> stateHandles = new HashMap<>(1);
    stateHandles.put(new StateHandleID("state1"), stateHandle);
    IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle = new IncrementalRemoteKeyedStateHandle(UUID.randomUUID(), KeyGroupRange.EMPTY_KEY_GROUP_RANGE, 1, stateHandles, stateHandles, stateHandle);
    try (RocksDBStateDownloader rocksDBStateDownloader = new RocksDBStateDownloader(5)) {
        rocksDBStateDownloader.transferAllStateDataToDirectory(incrementalKeyedStateHandle, temporaryFolder.newFolder().toPath(), new CloseableRegistry());
        fail();
    } catch (Exception e) {
        assertEquals(expectedException, e);
    }
}

Example 7 with IncrementalRemoteKeyedStateHandle

use of org.apache.flink.runtime.state.IncrementalRemoteKeyedStateHandle in project flink by apache.

the class SchedulerUtilsTest method buildIncrementalHandle.

private IncrementalRemoteKeyedStateHandle buildIncrementalHandle(StateHandleID key, StreamStateHandle shared, UUID backendIdentifier) {
    StreamStateHandle meta = new ByteStreamStateHandle("meta", new byte[] { 1, 2, 3 });
    Map<StateHandleID, StreamStateHandle> sharedStateMap = new HashMap<>();
    sharedStateMap.put(key, shared);
    return new IncrementalRemoteKeyedStateHandle(backendIdentifier, KeyGroupRange.EMPTY_KEY_GROUP_RANGE, 1, sharedStateMap, emptyMap(), meta);
}

Example 8 with IncrementalRemoteKeyedStateHandle

use of org.apache.flink.runtime.state.IncrementalRemoteKeyedStateHandle in project flink by apache.

the class SchedulerUtilsTest method testSharedStateRegistration.

/**
 * Check that a {@link SharedStateRegistryFactory} used by {@link SchedulerUtils} registers
 * shared checkpoint state on restore.
 */
@Test
public void testSharedStateRegistration() throws Exception {
    UUID backendId = UUID.randomUUID();
    StateHandleID key = new StateHandleID("k0");
    StreamStateHandle handle = new ByteStreamStateHandle("h0", new byte[] { 1, 2, 3 });
    CheckpointRecoveryFactory recoveryFactory = buildRecoveryFactory(buildCheckpoint(buildIncrementalHandle(key, handle, backendId)));
    CompletedCheckpointStore checkpointStore = SchedulerUtils.createCompletedCheckpointStore(new Configuration(), recoveryFactory, Executors.directExecutor(), log, new JobID());
    SharedStateRegistry sharedStateRegistry = checkpointStore.getSharedStateRegistry();
    IncrementalRemoteKeyedStateHandle newHandle = buildIncrementalHandle(key, new PlaceholderStreamStateHandle(1L), backendId);
    newHandle.registerSharedStates(sharedStateRegistry, 1L);
    assertSame(handle, newHandle.getSharedState().get(key));
}

Example 9 with IncrementalRemoteKeyedStateHandle

use of org.apache.flink.runtime.state.IncrementalRemoteKeyedStateHandle in project flink by apache.

the class RocksIncrementalSnapshotStrategyTest method testCheckpointIsIncremental.

// Verify the next checkpoint is still incremental after a savepoint completed.
@Test
public void testCheckpointIsIncremental() throws Exception {
    try (CloseableRegistry closeableRegistry = new CloseableRegistry();
        RocksIncrementalSnapshotStrategy checkpointSnapshotStrategy = createSnapshotStrategy(closeableRegistry)) {
        FsCheckpointStreamFactory checkpointStreamFactory = createFsCheckpointStreamFactory();
        // make and notify checkpoint with id 1
        snapshot(1L, checkpointSnapshotStrategy, checkpointStreamFactory, closeableRegistry);
        checkpointSnapshotStrategy.notifyCheckpointComplete(1L);
        // notify savepoint with id 2
        checkpointSnapshotStrategy.notifyCheckpointComplete(2L);
        // make checkpoint with id 3
        IncrementalRemoteKeyedStateHandle incrementalRemoteKeyedStateHandle3 = snapshot(3L, checkpointSnapshotStrategy, checkpointStreamFactory, closeableRegistry);
        // If 3rd checkpoint's placeholderStateHandleCount > 0,it means 3rd checkpoint is
        // incremental.
        Map<StateHandleID, StreamStateHandle> sharedState3 = incrementalRemoteKeyedStateHandle3.getSharedState();
        long placeholderStateHandleCount = sharedState3.entrySet().stream().filter(e -> e.getValue() instanceof PlaceholderStreamStateHandle).count();
        Assert.assertTrue(placeholderStateHandleCount > 0);
    }
}

Example 10 with IncrementalRemoteKeyedStateHandle

use of org.apache.flink.runtime.state.IncrementalRemoteKeyedStateHandle in project flink by apache.

the class EmbeddedRocksDBStateBackendTest method testSharedIncrementalStateDeRegistration.

@Test
public void testSharedIncrementalStateDeRegistration() throws Exception {
    if (enableIncrementalCheckpointing) {
        CheckpointableKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
        try {
            ValueStateDescriptor<String> kvId = new ValueStateDescriptor<>("id", String.class, null);
            kvId.initializeSerializerUnlessSet(new ExecutionConfig());
            ValueState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
            Queue<IncrementalRemoteKeyedStateHandle> previousStateHandles = new LinkedList<>();
            SharedStateRegistry sharedStateRegistry = spy(new SharedStateRegistryImpl());
            for (int checkpointId = 0; checkpointId < 3; ++checkpointId) {
                reset(sharedStateRegistry);
                backend.setCurrentKey(checkpointId);
                state.update("Hello-" + checkpointId);
                RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = backend.snapshot(checkpointId, checkpointId, createStreamFactory(), CheckpointOptions.forCheckpointWithDefaultLocation());
                snapshot.run();
                SnapshotResult<KeyedStateHandle> snapshotResult = snapshot.get();
                IncrementalRemoteKeyedStateHandle stateHandle = (IncrementalRemoteKeyedStateHandle) snapshotResult.getJobManagerOwnedSnapshot();
                Map<StateHandleID, StreamStateHandle> sharedState = new HashMap<>(stateHandle.getSharedState());
                stateHandle.registerSharedStates(sharedStateRegistry, checkpointId);
                for (Map.Entry<StateHandleID, StreamStateHandle> e : sharedState.entrySet()) {
                    verify(sharedStateRegistry).registerReference(stateHandle.createSharedStateRegistryKeyFromFileName(e.getKey()), e.getValue(), checkpointId);
                }
                previousStateHandles.add(stateHandle);
                ((CheckpointListener) backend).notifyCheckpointComplete(checkpointId);
                if (previousStateHandles.size() > 1) {
                    previousStateHandles.remove().discardState();
                }
            }
            while (!previousStateHandles.isEmpty()) {
                reset(sharedStateRegistry);
                previousStateHandles.remove().discardState();
            }
        } finally {
            IOUtils.closeQuietly(backend);
            backend.dispose();
        }
    }
}