Examples with KeyedStateHandle org.apache.flink.runtime.state.KeyedStateHandle used on opensource projects

Example 36 with KeyedStateHandle

use of org.apache.flink.runtime.state.KeyedStateHandle in project flink by apache.

the class StateBackendTestContext method dispose.

void dispose() throws Exception {
    disposeKeyedStateBackend();
    for (KeyedStateHandle snapshot : snapshots) {
        snapshot.discardState();
    }
    snapshots.clear();
    sharedStateRegistry.close();
    if (env != null) {
        env.close();
    }
}

Example 37 with KeyedStateHandle

use of org.apache.flink.runtime.state.KeyedStateHandle in project flink by apache.

the class TtlStateTestBase method testIncrementalCleanup.

@Test
public void testIncrementalCleanup() throws Exception {
    assumeTrue(incrementalCleanupSupported());
    initTest(getConfBuilder(TTL).cleanupIncrementally(5, true).build());
    final int keysToUpdate = (CopyOnWriteStateMap.DEFAULT_CAPACITY >> 3) * NUMBER_OF_KEY_GROUPS;
    timeProvider.time = 0;
    // create enough keys to trigger incremental rehash
    updateKeys(0, INC_CLEANUP_ALL_KEYS, ctx().updateEmpty);
    timeProvider.time = 50;
    // update some
    updateKeys(0, keysToUpdate, ctx().updateUnexpired);
    RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshotRunnableFuture = sbetc.triggerSnapshot();
    // update more concurrently with snapshotting
    updateKeys(keysToUpdate, keysToUpdate * 2, ctx().updateUnexpired);
    // expire rest
    timeProvider.time = 120;
    triggerMoreIncrementalCleanupByOtherOps();
    // check rest expired and cleanup updated
    checkExpiredKeys(keysToUpdate * 2, INC_CLEANUP_ALL_KEYS);
    KeyedStateHandle snapshot = snapshotRunnableFuture.get().getJobManagerOwnedSnapshot();
    // restore snapshot which should discard concurrent updates
    timeProvider.time = 50;
    restoreSnapshot(snapshot, NUMBER_OF_KEY_GROUPS);
    // check rest unexpired, also after restore which should discard concurrent updates
    checkUnexpiredKeys(keysToUpdate, INC_CLEANUP_ALL_KEYS, ctx().getUpdateEmpty);
    timeProvider.time = 120;
    // remove some
    for (int i = keysToUpdate >> 1; i < keysToUpdate + (keysToUpdate >> 2); i++) {
        sbetc.setCurrentKey(Integer.toString(i));
        ctx().ttlState.clear();
    }
    // check updated not expired
    checkUnexpiredKeys(0, keysToUpdate >> 1, ctx().getUnexpired);
    triggerMoreIncrementalCleanupByOtherOps();
    // check that concurrently updated and then restored with original values are expired
    checkExpiredKeys(keysToUpdate, keysToUpdate * 2);
    timeProvider.time = 170;
    // check rest expired and cleanup updated
    checkExpiredKeys(keysToUpdate >> 1, INC_CLEANUP_ALL_KEYS);
    // check updated expired
    checkExpiredKeys(0, keysToUpdate >> 1);
}

Example 38 with KeyedStateHandle

use of org.apache.flink.runtime.state.KeyedStateHandle in project flink by apache.

the class RocksDBIncrementalCheckpointUtils method chooseTheBestStateHandleForInitial.

/**
 * Choose the best state handle according to the {@link #STATE_HANDLE_EVALUATOR} to init the
 * initial db.
 *
 * @param restoreStateHandles The candidate state handles.
 * @param targetKeyGroupRange The target key group range.
 * @return The best candidate or null if no candidate was a good fit.
 */
@Nullable
public static KeyedStateHandle chooseTheBestStateHandleForInitial(@Nonnull Collection<KeyedStateHandle> restoreStateHandles, @Nonnull KeyGroupRange targetKeyGroupRange) {
    KeyedStateHandle bestStateHandle = null;
    Score bestScore = Score.MIN;
    for (KeyedStateHandle rawStateHandle : restoreStateHandles) {
        Score handleScore = STATE_HANDLE_EVALUATOR.apply(rawStateHandle, targetKeyGroupRange);
        if (handleScore.compareTo(bestScore) > 0) {
            bestStateHandle = rawStateHandle;
            bestScore = handleScore;
        }
    }
    return bestStateHandle;
}

Example 39 with KeyedStateHandle

use of org.apache.flink.runtime.state.KeyedStateHandle in project flink by apache.

the class EmbeddedRocksDBStateBackendTest method testSharedIncrementalStateDeRegistration.

@Test
public void testSharedIncrementalStateDeRegistration() throws Exception {
    if (enableIncrementalCheckpointing) {
        CheckpointableKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
        try {
            ValueStateDescriptor<String> kvId = new ValueStateDescriptor<>("id", String.class, null);
            kvId.initializeSerializerUnlessSet(new ExecutionConfig());
            ValueState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
            Queue<IncrementalRemoteKeyedStateHandle> previousStateHandles = new LinkedList<>();
            SharedStateRegistry sharedStateRegistry = spy(new SharedStateRegistryImpl());
            for (int checkpointId = 0; checkpointId < 3; ++checkpointId) {
                reset(sharedStateRegistry);
                backend.setCurrentKey(checkpointId);
                state.update("Hello-" + checkpointId);
                RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = backend.snapshot(checkpointId, checkpointId, createStreamFactory(), CheckpointOptions.forCheckpointWithDefaultLocation());
                snapshot.run();
                SnapshotResult<KeyedStateHandle> snapshotResult = snapshot.get();
                IncrementalRemoteKeyedStateHandle stateHandle = (IncrementalRemoteKeyedStateHandle) snapshotResult.getJobManagerOwnedSnapshot();
                Map<StateHandleID, StreamStateHandle> sharedState = new HashMap<>(stateHandle.getSharedState());
                stateHandle.registerSharedStates(sharedStateRegistry, checkpointId);
                for (Map.Entry<StateHandleID, StreamStateHandle> e : sharedState.entrySet()) {
                    verify(sharedStateRegistry).registerReference(stateHandle.createSharedStateRegistryKeyFromFileName(e.getKey()), e.getValue(), checkpointId);
                }
                previousStateHandles.add(stateHandle);
                ((CheckpointListener) backend).notifyCheckpointComplete(checkpointId);
                if (previousStateHandles.size() > 1) {
                    previousStateHandles.remove().discardState();
                }
            }
            while (!previousStateHandles.isEmpty()) {
                reset(sharedStateRegistry);
                previousStateHandles.remove().discardState();
            }
        } finally {
            IOUtils.closeQuietly(backend);
            backend.dispose();
        }
    }
}

Example 40 with KeyedStateHandle

use of org.apache.flink.runtime.state.KeyedStateHandle in project flink by apache.

the class EmbeddedRocksDBStateBackendTest method testCompletingSnapshot.

@Test
public void testCompletingSnapshot() throws Exception {
    setupRocksKeyedStateBackend();
    try {
        RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = keyedStateBackend.snapshot(0L, 0L, testStreamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
        Thread asyncSnapshotThread = new Thread(snapshot);
        asyncSnapshotThread.start();
        // wait for snapshot to run
        waiter.await();
        waiter.reset();
        runStateUpdates();
        // allow checkpointing to start writing
        blocker.trigger();
        // wait for snapshot stream writing to run
        waiter.await();
        SnapshotResult<KeyedStateHandle> snapshotResult = snapshot.get();
        KeyedStateHandle keyedStateHandle = snapshotResult.getJobManagerOwnedSnapshot();
        assertNotNull(keyedStateHandle);
        assertTrue(keyedStateHandle.getStateSize() > 0);
        assertEquals(2, keyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
        for (BlockingCheckpointOutputStream stream : testStreamFactory.getAllCreatedStreams()) {
            assertTrue(stream.isClosed());
        }
        asyncSnapshotThread.join();
        verifyRocksObjectsReleased();
    } finally {
        this.keyedStateBackend.dispose();
        this.keyedStateBackend = null;
    }
    verifyRocksDBStateUploaderClosed();
}