Examples with ChangelogStateBackendHandle org.apache.flink.runtime.state.changelog.ChangelogStateBackendHandle used on opensource projects

Example 1 with ChangelogStateBackendHandle

use of org.apache.flink.runtime.state.changelog.ChangelogStateBackendHandle in project flink by apache.

the class MetadataV3SerializerTest method testSerializeChangelogStateBackendHandle.

private void testSerializeChangelogStateBackendHandle(boolean fullSnapshot) throws IOException {
    ChangelogStateBackendHandle handle = createChangelogStateBackendHandle(fullSnapshot);
    try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
        MetadataV2V3SerializerBase.serializeKeyedStateHandle(handle, new DataOutputStream(out));
        try (ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray())) {
            KeyedStateHandle deserialized = MetadataV2V3SerializerBase.deserializeKeyedStateHandle(new DataInputStream(in), null);
            assertTrue(deserialized instanceof ChangelogStateBackendHandleImpl);
            assertEquals(((ChangelogStateBackendHandleImpl) deserialized).getMaterializedStateHandles(), handle.getMaterializedStateHandles());
        }
    }
}

Example 2 with ChangelogStateBackendHandle

use of org.apache.flink.runtime.state.changelog.ChangelogStateBackendHandle in project flink by apache.

the class MetadataV2V3SerializerBase method serializeKeyedStateHandle.

// ------------------------------------------------------------------------
// keyed state
// ------------------------------------------------------------------------
@VisibleForTesting
static void serializeKeyedStateHandle(KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {
    if (stateHandle == null) {
        dos.writeByte(NULL_HANDLE);
    } else if (stateHandle instanceof KeyGroupsStateHandle) {
        KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;
        if (stateHandle instanceof KeyGroupsSavepointStateHandle) {
            dos.writeByte(SAVEPOINT_KEY_GROUPS_HANDLE);
        } else {
            dos.writeByte(KEY_GROUPS_HANDLE_V2);
        }
        dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
        dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
        for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
            dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
        }
        serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
        // savepoint state handle would not need to persist state handle id out.
        if (!(stateHandle instanceof KeyGroupsSavepointStateHandle)) {
            writeStateHandleId(stateHandle, dos);
        }
    } else if (stateHandle instanceof IncrementalRemoteKeyedStateHandle) {
        IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle = (IncrementalRemoteKeyedStateHandle) stateHandle;
        dos.writeByte(INCREMENTAL_KEY_GROUPS_HANDLE_V2);
        dos.writeLong(incrementalKeyedStateHandle.getCheckpointId());
        dos.writeUTF(String.valueOf(incrementalKeyedStateHandle.getBackendIdentifier()));
        dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getStartKeyGroup());
        dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
        dos.writeLong(incrementalKeyedStateHandle.getCheckpointedSize());
        serializeStreamStateHandle(incrementalKeyedStateHandle.getMetaStateHandle(), dos);
        serializeStreamStateHandleMap(incrementalKeyedStateHandle.getSharedState(), dos);
        serializeStreamStateHandleMap(incrementalKeyedStateHandle.getPrivateState(), dos);
        writeStateHandleId(incrementalKeyedStateHandle, dos);
    } else if (stateHandle instanceof ChangelogStateBackendHandle) {
        ChangelogStateBackendHandle handle = (ChangelogStateBackendHandle) stateHandle;
        dos.writeByte(CHANGELOG_HANDLE);
        dos.writeInt(handle.getKeyGroupRange().getStartKeyGroup());
        dos.writeInt(handle.getKeyGroupRange().getNumberOfKeyGroups());
        dos.writeLong(handle.getCheckpointedSize());
        dos.writeInt(handle.getMaterializedStateHandles().size());
        for (KeyedStateHandle keyedStateHandle : handle.getMaterializedStateHandles()) {
            serializeKeyedStateHandle(keyedStateHandle, dos);
        }
        dos.writeInt(handle.getNonMaterializedStateHandles().size());
        for (KeyedStateHandle k : handle.getNonMaterializedStateHandles()) {
            serializeKeyedStateHandle(k, dos);
        }
        dos.writeLong(handle.getMaterializationID());
        writeStateHandleId(handle, dos);
    } else if (stateHandle instanceof InMemoryChangelogStateHandle) {
        InMemoryChangelogStateHandle handle = (InMemoryChangelogStateHandle) stateHandle;
        dos.writeByte(CHANGELOG_BYTE_INCREMENT_HANDLE);
        dos.writeInt(handle.getKeyGroupRange().getStartKeyGroup());
        dos.writeInt(handle.getKeyGroupRange().getNumberOfKeyGroups());
        dos.writeLong(handle.getFrom());
        dos.writeLong(handle.getTo());
        dos.writeInt(handle.getChanges().size());
        for (StateChange change : handle.getChanges()) {
            dos.writeInt(change.getKeyGroup());
            dos.writeInt(change.getChange().length);
            dos.write(change.getChange());
        }
        writeStateHandleId(handle, dos);
    } else if (stateHandle instanceof ChangelogStateHandleStreamImpl) {
        ChangelogStateHandleStreamImpl handle = (ChangelogStateHandleStreamImpl) stateHandle;
        dos.writeByte(CHANGELOG_FILE_INCREMENT_HANDLE);
        dos.writeInt(handle.getKeyGroupRange().getStartKeyGroup());
        dos.writeInt(handle.getKeyGroupRange().getNumberOfKeyGroups());
        dos.writeInt(handle.getHandlesAndOffsets().size());
        for (Tuple2<StreamStateHandle, Long> streamHandleAndOffset : handle.getHandlesAndOffsets()) {
            dos.writeLong(streamHandleAndOffset.f1);
            serializeStreamStateHandle(streamHandleAndOffset.f0, dos);
        }
        dos.writeLong(handle.getStateSize());
        dos.writeLong(handle.getCheckpointedSize());
        writeStateHandleId(handle, dos);
    } else {
        throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
    }
}

Example 3 with ChangelogStateBackendHandle

use of org.apache.flink.runtime.state.changelog.ChangelogStateBackendHandle in project flink by apache.

the class ChangelogStateBackend method restore.

@SuppressWarnings({ "unchecked", "rawtypes" })
private <K> ChangelogKeyedStateBackend<K> restore(Environment env, String operatorIdentifier, KeyGroupRange keyGroupRange, TtlTimeProvider ttlTimeProvider, Collection<KeyedStateHandle> stateHandles, BaseBackendBuilder<K> baseBackendBuilder) throws Exception {
    StateChangelogStorage<?> changelogStorage = Preconditions.checkNotNull(env.getTaskStateManager().getStateChangelogStorage(), "Changelog storage is null when creating and restoring" + " the ChangelogKeyedStateBackend.");
    String subtaskName = env.getTaskInfo().getTaskNameWithSubtasks();
    ExecutionConfig executionConfig = env.getExecutionConfig();
    Collection<ChangelogStateBackendHandle> stateBackendHandles = castHandles(stateHandles);
    ChangelogKeyedStateBackend<K> keyedStateBackend = ChangelogBackendRestoreOperation.restore(changelogStorage.createReader(), env.getUserCodeClassLoader().asClassLoader(), stateBackendHandles, baseBackendBuilder, (baseBackend, baseState) -> new ChangelogKeyedStateBackend(baseBackend, subtaskName, executionConfig, ttlTimeProvider, changelogStorage.createWriter(operatorIdentifier, keyGroupRange), baseState, env.getCheckpointStorageAccess()));
    PeriodicMaterializationManager periodicMaterializationManager = new PeriodicMaterializationManager(checkNotNull(env.getMainMailboxExecutor()), checkNotNull(env.getAsyncOperationsThreadPool()), subtaskName, (message, exception) -> env.failExternally(new AsynchronousException(message, exception)), keyedStateBackend, executionConfig.getPeriodicMaterializeIntervalMillis(), executionConfig.getMaterializationMaxAllowedFailures());
    // keyedStateBackend is responsible to close periodicMaterializationManager
    // This indicates periodicMaterializationManager binds to the keyedStateBackend
    // However PeriodicMaterializationManager can not be part of keyedStateBackend
    // because of cyclic reference
    keyedStateBackend.registerCloseable(periodicMaterializationManager);
    periodicMaterializationManager.start();
    return keyedStateBackend;
}

Example 4 with ChangelogStateBackendHandle

use of org.apache.flink.runtime.state.changelog.ChangelogStateBackendHandle in project flink by apache.

the class ChangelogKeyedStateBackend method completeRestore.

private ChangelogSnapshotState completeRestore(Collection<ChangelogStateBackendHandle> stateHandles) {
    long materializationId = 0L;
    List<KeyedStateHandle> materialized = new ArrayList<>();
    List<ChangelogStateHandle> restoredNonMaterialized = new ArrayList<>();
    for (ChangelogStateBackendHandle h : stateHandles) {
        if (h != null) {
            materialized.addAll(h.getMaterializedStateHandles());
            restoredNonMaterialized.addAll(h.getNonMaterializedStateHandles());
            // choose max materializationID to handle rescaling
            materializationId = Math.max(materializationId, h.getMaterializationID());
        }
    }
    this.materializedId = materializationId + 1;
    return new ChangelogSnapshotState(materialized, restoredNonMaterialized, stateChangelogWriter.initialSequenceNumber(), materializationId);
}

Example 5 with ChangelogStateBackendHandle

use of org.apache.flink.runtime.state.changelog.ChangelogStateBackendHandle in project flink by apache.

the class ChangelogBackendRestoreOperation method restore.

public static <K, T extends ChangelogStateHandle> ChangelogKeyedStateBackend<K> restore(StateChangelogHandleReader<T> changelogHandleReader, ClassLoader classLoader, Collection<ChangelogStateBackendHandle> stateHandles, BaseBackendBuilder<K> baseBackendBuilder, DeltaBackendBuilder<K> changelogBackendBuilder) throws Exception {
    Collection<KeyedStateHandle> baseState = extractBaseState(stateHandles);
    AbstractKeyedStateBackend<K> baseBackend = baseBackendBuilder.apply(baseState);
    ChangelogKeyedStateBackend<K> changelogBackend = changelogBackendBuilder.apply(baseBackend, stateHandles);
    for (ChangelogStateBackendHandle handle : stateHandles) {
        if (handle != null) {
            // null is empty state (no change)
            readBackendHandle(changelogBackend, handle, changelogHandleReader, classLoader);
        }
    }
    return changelogBackend;
}