Examples with ChangelogStateHandle org.apache.flink.runtime.state.changelog.ChangelogStateHandle used on opensource projects

Example 1 with ChangelogStateHandle

use of org.apache.flink.runtime.state.changelog.ChangelogStateHandle in project flink by apache.

the class ChangelogKeyedStateBackend method buildSnapshotResult.

private SnapshotResult<KeyedStateHandle> buildSnapshotResult(ChangelogStateHandle delta, ChangelogSnapshotState changelogStateBackendStateCopy) {
    // collections don't change once started and handles are immutable
    List<ChangelogStateHandle> prevDeltaCopy = new ArrayList<>(changelogStateBackendStateCopy.getRestoredNonMaterialized());
    long persistedSizeOfThisCheckpoint = 0L;
    if (delta != null && delta.getStateSize() > 0) {
        prevDeltaCopy.add(delta);
        persistedSizeOfThisCheckpoint += delta.getCheckpointedSize();
    }
    if (prevDeltaCopy.isEmpty() && changelogStateBackendStateCopy.getMaterializedSnapshot().isEmpty()) {
        return SnapshotResult.empty();
    } else {
        return SnapshotResult.of(new ChangelogStateBackendHandleImpl(changelogStateBackendStateCopy.getMaterializedSnapshot(), prevDeltaCopy, getKeyGroupRange(), changelogStateBackendStateCopy.materializationID, persistedSizeOfThisCheckpoint));
    }
}

Example 2 with ChangelogStateHandle

use of org.apache.flink.runtime.state.changelog.ChangelogStateHandle in project flink by apache.

the class MetadataV2V3SerializerBase method deserializeKeyedStateHandle.

@VisibleForTesting
@Nullable
static KeyedStateHandle deserializeKeyedStateHandle(DataInputStream dis, @Nullable DeserializationContext context) throws IOException {
    final int type = dis.readByte();
    if (NULL_HANDLE == type) {
        return null;
    } else if (KEY_GROUPS_HANDLE == type || KEY_GROUPS_HANDLE_V2 == type || SAVEPOINT_KEY_GROUPS_HANDLE == type) {
        int startKeyGroup = dis.readInt();
        int numKeyGroups = dis.readInt();
        KeyGroupRange keyGroupRange = KeyGroupRange.of(startKeyGroup, startKeyGroup + numKeyGroups - 1);
        long[] offsets = new long[numKeyGroups];
        for (int i = 0; i < numKeyGroups; ++i) {
            offsets[i] = dis.readLong();
        }
        KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(keyGroupRange, offsets);
        StreamStateHandle stateHandle = deserializeStreamStateHandle(dis, context);
        if (SAVEPOINT_KEY_GROUPS_HANDLE == type) {
            return new KeyGroupsSavepointStateHandle(keyGroupRangeOffsets, stateHandle);
        } else {
            StateHandleID stateHandleID = KEY_GROUPS_HANDLE_V2 == type ? new StateHandleID(dis.readUTF()) : StateHandleID.randomStateHandleId();
            return KeyGroupsStateHandle.restore(keyGroupRangeOffsets, stateHandle, stateHandleID);
        }
    } else if (INCREMENTAL_KEY_GROUPS_HANDLE == type || INCREMENTAL_KEY_GROUPS_HANDLE_V2 == type) {
        return deserializeIncrementalStateHandle(dis, context, type);
    } else if (CHANGELOG_HANDLE == type) {
        int startKeyGroup = dis.readInt();
        int numKeyGroups = dis.readInt();
        KeyGroupRange keyGroupRange = KeyGroupRange.of(startKeyGroup, startKeyGroup + numKeyGroups - 1);
        long checkpointedSize = dis.readLong();
        int baseSize = dis.readInt();
        List<KeyedStateHandle> base = new ArrayList<>(baseSize);
        for (int i = 0; i < baseSize; i++) {
            KeyedStateHandle handle = deserializeKeyedStateHandle(dis, context);
            if (handle != null) {
                base.add(handle);
            } else {
                LOG.warn("Unexpected null keyed state handle of materialized part when deserializing changelog state-backend handle");
            }
        }
        int deltaSize = dis.readInt();
        List<ChangelogStateHandle> delta = new ArrayList<>(deltaSize);
        for (int i = 0; i < deltaSize; i++) {
            delta.add((ChangelogStateHandle) deserializeKeyedStateHandle(dis, context));
        }
        long materializationID = dis.readLong();
        StateHandleID stateHandleId = new StateHandleID(dis.readUTF());
        return ChangelogStateBackendHandleImpl.restore(base, delta, keyGroupRange, materializationID, checkpointedSize, stateHandleId);
    } else if (CHANGELOG_BYTE_INCREMENT_HANDLE == type) {
        int start = dis.readInt();
        int numKeyGroups = dis.readInt();
        KeyGroupRange keyGroupRange = KeyGroupRange.of(start, start + numKeyGroups - 1);
        long from = dis.readLong();
        long to = dis.readLong();
        int size = dis.readInt();
        List<StateChange> changes = new ArrayList<>(size);
        for (int i = 0; i < size; i++) {
            int keyGroup = dis.readInt();
            int bytesSize = dis.readInt();
            byte[] bytes = new byte[bytesSize];
            IOUtils.readFully(dis, bytes, 0, bytesSize);
            changes.add(new StateChange(keyGroup, bytes));
        }
        StateHandleID stateHandleId = new StateHandleID(dis.readUTF());
        return InMemoryChangelogStateHandle.restore(changes, SequenceNumber.of(from), SequenceNumber.of(to), keyGroupRange, stateHandleId);
    } else if (CHANGELOG_FILE_INCREMENT_HANDLE == type) {
        int start = dis.readInt();
        int numKeyGroups = dis.readInt();
        KeyGroupRange keyGroupRange = KeyGroupRange.of(start, start + numKeyGroups - 1);
        int numHandles = dis.readInt();
        List<Tuple2<StreamStateHandle, Long>> streamHandleAndOffset = new ArrayList<>(numHandles);
        for (int i = 0; i < numHandles; i++) {
            long o = dis.readLong();
            StreamStateHandle h = deserializeStreamStateHandle(dis, context);
            streamHandleAndOffset.add(Tuple2.of(h, o));
        }
        long size = dis.readLong();
        long checkpointedSize = dis.readLong();
        StateHandleID stateHandleId = new StateHandleID(dis.readUTF());
        return ChangelogStateHandleStreamImpl.restore(streamHandleAndOffset, keyGroupRange, size, checkpointedSize, stateHandleId);
    } else {
        throw new IllegalStateException("Reading invalid KeyedStateHandle, type: " + type);
    }
}

Example 3 with ChangelogStateHandle

use of org.apache.flink.runtime.state.changelog.ChangelogStateHandle in project flink by apache.

the class ChangelogKeyedStateBackend method completeRestore.

private ChangelogSnapshotState completeRestore(Collection<ChangelogStateBackendHandle> stateHandles) {
    long materializationId = 0L;
    List<KeyedStateHandle> materialized = new ArrayList<>();
    List<ChangelogStateHandle> restoredNonMaterialized = new ArrayList<>();
    for (ChangelogStateBackendHandle h : stateHandles) {
        if (h != null) {
            materialized.addAll(h.getMaterializedStateHandles());
            restoredNonMaterialized.addAll(h.getNonMaterializedStateHandles());
            // choose max materializationID to handle rescaling
            materializationId = Math.max(materializationId, h.getMaterializationID());
        }
    }
    this.materializedId = materializationId + 1;
    return new ChangelogSnapshotState(materialized, restoredNonMaterialized, stateChangelogWriter.initialSequenceNumber(), materializationId);
}