Examples with KeyedStateHandle org.apache.flink.runtime.state.KeyedStateHandle used on opensource projects

Example 6 with KeyedStateHandle

use of org.apache.flink.runtime.state.KeyedStateHandle in project flink by apache.

the class MetadataV2V3SerializerBase method serializeKeyedStateHandle.

// ------------------------------------------------------------------------
// keyed state
// ------------------------------------------------------------------------
@VisibleForTesting
static void serializeKeyedStateHandle(KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {
    if (stateHandle == null) {
        dos.writeByte(NULL_HANDLE);
    } else if (stateHandle instanceof KeyGroupsStateHandle) {
        KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;
        if (stateHandle instanceof KeyGroupsSavepointStateHandle) {
            dos.writeByte(SAVEPOINT_KEY_GROUPS_HANDLE);
        } else {
            dos.writeByte(KEY_GROUPS_HANDLE_V2);
        }
        dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
        dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
        for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
            dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
        }
        serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
        // savepoint state handle would not need to persist state handle id out.
        if (!(stateHandle instanceof KeyGroupsSavepointStateHandle)) {
            writeStateHandleId(stateHandle, dos);
        }
    } else if (stateHandle instanceof IncrementalRemoteKeyedStateHandle) {
        IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle = (IncrementalRemoteKeyedStateHandle) stateHandle;
        dos.writeByte(INCREMENTAL_KEY_GROUPS_HANDLE_V2);
        dos.writeLong(incrementalKeyedStateHandle.getCheckpointId());
        dos.writeUTF(String.valueOf(incrementalKeyedStateHandle.getBackendIdentifier()));
        dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getStartKeyGroup());
        dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
        dos.writeLong(incrementalKeyedStateHandle.getCheckpointedSize());
        serializeStreamStateHandle(incrementalKeyedStateHandle.getMetaStateHandle(), dos);
        serializeStreamStateHandleMap(incrementalKeyedStateHandle.getSharedState(), dos);
        serializeStreamStateHandleMap(incrementalKeyedStateHandle.getPrivateState(), dos);
        writeStateHandleId(incrementalKeyedStateHandle, dos);
    } else if (stateHandle instanceof ChangelogStateBackendHandle) {
        ChangelogStateBackendHandle handle = (ChangelogStateBackendHandle) stateHandle;
        dos.writeByte(CHANGELOG_HANDLE);
        dos.writeInt(handle.getKeyGroupRange().getStartKeyGroup());
        dos.writeInt(handle.getKeyGroupRange().getNumberOfKeyGroups());
        dos.writeLong(handle.getCheckpointedSize());
        dos.writeInt(handle.getMaterializedStateHandles().size());
        for (KeyedStateHandle keyedStateHandle : handle.getMaterializedStateHandles()) {
            serializeKeyedStateHandle(keyedStateHandle, dos);
        }
        dos.writeInt(handle.getNonMaterializedStateHandles().size());
        for (KeyedStateHandle k : handle.getNonMaterializedStateHandles()) {
            serializeKeyedStateHandle(k, dos);
        }
        dos.writeLong(handle.getMaterializationID());
        writeStateHandleId(handle, dos);
    } else if (stateHandle instanceof InMemoryChangelogStateHandle) {
        InMemoryChangelogStateHandle handle = (InMemoryChangelogStateHandle) stateHandle;
        dos.writeByte(CHANGELOG_BYTE_INCREMENT_HANDLE);
        dos.writeInt(handle.getKeyGroupRange().getStartKeyGroup());
        dos.writeInt(handle.getKeyGroupRange().getNumberOfKeyGroups());
        dos.writeLong(handle.getFrom());
        dos.writeLong(handle.getTo());
        dos.writeInt(handle.getChanges().size());
        for (StateChange change : handle.getChanges()) {
            dos.writeInt(change.getKeyGroup());
            dos.writeInt(change.getChange().length);
            dos.write(change.getChange());
        }
        writeStateHandleId(handle, dos);
    } else if (stateHandle instanceof ChangelogStateHandleStreamImpl) {
        ChangelogStateHandleStreamImpl handle = (ChangelogStateHandleStreamImpl) stateHandle;
        dos.writeByte(CHANGELOG_FILE_INCREMENT_HANDLE);
        dos.writeInt(handle.getKeyGroupRange().getStartKeyGroup());
        dos.writeInt(handle.getKeyGroupRange().getNumberOfKeyGroups());
        dos.writeInt(handle.getHandlesAndOffsets().size());
        for (Tuple2<StreamStateHandle, Long> streamHandleAndOffset : handle.getHandlesAndOffsets()) {
            dos.writeLong(streamHandleAndOffset.f1);
            serializeStreamStateHandle(streamHandleAndOffset.f0, dos);
        }
        dos.writeLong(handle.getStateSize());
        dos.writeLong(handle.getCheckpointedSize());
        writeStateHandleId(handle, dos);
    } else {
        throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
    }
}

Example 7 with KeyedStateHandle

use of org.apache.flink.runtime.state.KeyedStateHandle in project flink by apache.

the class StateAssignmentOperationTest method assigningStateHandlesCanNotBeNull.

@Test
public void assigningStateHandlesCanNotBeNull() {
    OperatorState state = new OperatorState(new OperatorID(), 1, MAX_P);
    List<KeyedStateHandle> managedKeyedStateHandles = StateAssignmentOperation.getManagedKeyedStateHandles(state, KeyGroupRange.of(0, 1));
    List<KeyedStateHandle> rawKeyedStateHandles = StateAssignmentOperation.getRawKeyedStateHandles(state, KeyGroupRange.of(0, 1));
    assertThat(managedKeyedStateHandles, is(empty()));
    assertThat(rawKeyedStateHandles, is(empty()));
}

Example 8 with KeyedStateHandle

use of org.apache.flink.runtime.state.KeyedStateHandle in project flink by apache.

the class HeapKeyedStateBackendSnapshotMigrationTest method testMapStateMigrationAfterHashMapSerRemoval.

@Test
public void testMapStateMigrationAfterHashMapSerRemoval() throws Exception {
    ClassLoader cl = getClass().getClassLoader();
    URL resource = cl.getResource("heap_keyed_statebackend_1_5_map.snapshot");
    Preconditions.checkNotNull(resource, "Binary snapshot resource not found!");
    final SnapshotResult<KeyedStateHandle> stateHandles;
    try (BufferedInputStream bis = new BufferedInputStream((new FileInputStream(resource.getFile())))) {
        stateHandles = InstantiationUtil.deserializeObject(bis, Thread.currentThread().getContextClassLoader());
    }
    final KeyedStateHandle stateHandle = stateHandles.getJobManagerOwnedSnapshot();
    try (final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend(StateObjectCollection.singleton(stateHandle))) {
        final Integer namespace1 = 1;
        final Integer namespace2 = 2;
        final Integer namespace3 = 3;
        final MapStateDescriptor<Long, Long> stateDescr = new MapStateDescriptor<>("my-map-state", Long.class, Long.class);
        stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());
        InternalMapState<String, Integer, Long, Long> state = keyedBackend.createInternalState(IntSerializer.INSTANCE, stateDescr);
        keyedBackend.setCurrentKey("abc");
        state.setCurrentNamespace(namespace1);
        assertEquals(33L, (long) state.get(33L));
        assertEquals(55L, (long) state.get(55L));
        assertEquals(2, getStateSize(state));
        state.setCurrentNamespace(namespace2);
        assertEquals(22L, (long) state.get(22L));
        assertEquals(11L, (long) state.get(11L));
        assertEquals(2, getStateSize(state));
        state.setCurrentNamespace(namespace3);
        assertEquals(44L, (long) state.get(44L));
        assertEquals(1, getStateSize(state));
        keyedBackend.setCurrentKey("def");
        state.setCurrentNamespace(namespace1);
        assertEquals(11L, (long) state.get(11L));
        assertEquals(44L, (long) state.get(44L));
        assertEquals(2, getStateSize(state));
        state.setCurrentNamespace(namespace3);
        assertEquals(22L, (long) state.get(22L));
        assertEquals(55L, (long) state.get(55L));
        assertEquals(33L, (long) state.get(33L));
        assertEquals(3, getStateSize(state));
        keyedBackend.setCurrentKey("jkl");
        state.setCurrentNamespace(namespace1);
        assertEquals(11L, (long) state.get(11L));
        assertEquals(22L, (long) state.get(22L));
        assertEquals(33L, (long) state.get(33L));
        assertEquals(44L, (long) state.get(44L));
        assertEquals(55L, (long) state.get(55L));
        assertEquals(5, getStateSize(state));
        keyedBackend.setCurrentKey("mno");
        state.setCurrentNamespace(namespace3);
        assertEquals(11L, (long) state.get(11L));
        assertEquals(22L, (long) state.get(22L));
        assertEquals(33L, (long) state.get(33L));
        assertEquals(44L, (long) state.get(44L));
        assertEquals(55L, (long) state.get(55L));
        assertEquals(5, getStateSize(state));
        RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = keyedBackend.snapshot(1L, 1L, new MemCheckpointStreamFactory(4 * 1024 * 1024), CheckpointOptions.forCheckpointWithDefaultLocation());
        snapshot.run();
    }
}

Example 9 with KeyedStateHandle

use of org.apache.flink.runtime.state.KeyedStateHandle in project flink by apache.

the class StateBackendTestContext method takeSnapshot.

KeyedStateHandle takeSnapshot() throws Exception {
    SnapshotResult<KeyedStateHandle> snapshotResult = triggerSnapshot().get();
    KeyedStateHandle jobManagerOwnedSnapshot = snapshotResult.getJobManagerOwnedSnapshot();
    if (jobManagerOwnedSnapshot != null) {
        jobManagerOwnedSnapshot.registerSharedStates(sharedStateRegistry, 0L);
    }
    return jobManagerOwnedSnapshot;
}

Example 10 with KeyedStateHandle

use of org.apache.flink.runtime.state.KeyedStateHandle in project flink by apache.

the class TtlStateTestBase method testSnapshotChangeRestore.

@Test
public void testSnapshotChangeRestore() throws Exception {
    initTest();
    timeProvider.time = 0;
    sbetc.setCurrentKey("k1");
    ctx().update(ctx().updateEmpty);
    timeProvider.time = 50;
    sbetc.setCurrentKey("k1");
    ctx().update(ctx().updateUnexpired);
    timeProvider.time = 100;
    sbetc.setCurrentKey("k2");
    ctx().update(ctx().updateEmpty);
    KeyedStateHandle snapshot = sbetc.takeSnapshot();
    timeProvider.time = 170;
    sbetc.setCurrentKey("k1");
    ctx().update(ctx().updateExpired);
    sbetc.setCurrentKey("k2");
    ctx().update(ctx().updateUnexpired);
    restoreSnapshot(snapshot, NUMBER_OF_KEY_GROUPS);
    timeProvider.time = 180;
    sbetc.setCurrentKey("k1");
    assertEquals(EXPIRED_UNAVAIL, ctx().emptyValue, ctx().get());
    sbetc.setCurrentKey("k2");
    assertEquals(UNEXPIRED_AVAIL, ctx().getUpdateEmpty, ctx().get());
}