Examples with StateChange org.apache.flink.runtime.state.changelog.StateChange used on opensource projects

Example 1 with StateChange

use of org.apache.flink.runtime.state.changelog.StateChange in project flink by apache.

the class MetadataV2V3SerializerBase method serializeKeyedStateHandle.

// ------------------------------------------------------------------------
// keyed state
// ------------------------------------------------------------------------
@VisibleForTesting
static void serializeKeyedStateHandle(KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {
    if (stateHandle == null) {
        dos.writeByte(NULL_HANDLE);
    } else if (stateHandle instanceof KeyGroupsStateHandle) {
        KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;
        if (stateHandle instanceof KeyGroupsSavepointStateHandle) {
            dos.writeByte(SAVEPOINT_KEY_GROUPS_HANDLE);
        } else {
            dos.writeByte(KEY_GROUPS_HANDLE_V2);
        }
        dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
        dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
        for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
            dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
        }
        serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
        // savepoint state handle would not need to persist state handle id out.
        if (!(stateHandle instanceof KeyGroupsSavepointStateHandle)) {
            writeStateHandleId(stateHandle, dos);
        }
    } else if (stateHandle instanceof IncrementalRemoteKeyedStateHandle) {
        IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle = (IncrementalRemoteKeyedStateHandle) stateHandle;
        dos.writeByte(INCREMENTAL_KEY_GROUPS_HANDLE_V2);
        dos.writeLong(incrementalKeyedStateHandle.getCheckpointId());
        dos.writeUTF(String.valueOf(incrementalKeyedStateHandle.getBackendIdentifier()));
        dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getStartKeyGroup());
        dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
        dos.writeLong(incrementalKeyedStateHandle.getCheckpointedSize());
        serializeStreamStateHandle(incrementalKeyedStateHandle.getMetaStateHandle(), dos);
        serializeStreamStateHandleMap(incrementalKeyedStateHandle.getSharedState(), dos);
        serializeStreamStateHandleMap(incrementalKeyedStateHandle.getPrivateState(), dos);
        writeStateHandleId(incrementalKeyedStateHandle, dos);
    } else if (stateHandle instanceof ChangelogStateBackendHandle) {
        ChangelogStateBackendHandle handle = (ChangelogStateBackendHandle) stateHandle;
        dos.writeByte(CHANGELOG_HANDLE);
        dos.writeInt(handle.getKeyGroupRange().getStartKeyGroup());
        dos.writeInt(handle.getKeyGroupRange().getNumberOfKeyGroups());
        dos.writeLong(handle.getCheckpointedSize());
        dos.writeInt(handle.getMaterializedStateHandles().size());
        for (KeyedStateHandle keyedStateHandle : handle.getMaterializedStateHandles()) {
            serializeKeyedStateHandle(keyedStateHandle, dos);
        }
        dos.writeInt(handle.getNonMaterializedStateHandles().size());
        for (KeyedStateHandle k : handle.getNonMaterializedStateHandles()) {
            serializeKeyedStateHandle(k, dos);
        }
        dos.writeLong(handle.getMaterializationID());
        writeStateHandleId(handle, dos);
    } else if (stateHandle instanceof InMemoryChangelogStateHandle) {
        InMemoryChangelogStateHandle handle = (InMemoryChangelogStateHandle) stateHandle;
        dos.writeByte(CHANGELOG_BYTE_INCREMENT_HANDLE);
        dos.writeInt(handle.getKeyGroupRange().getStartKeyGroup());
        dos.writeInt(handle.getKeyGroupRange().getNumberOfKeyGroups());
        dos.writeLong(handle.getFrom());
        dos.writeLong(handle.getTo());
        dos.writeInt(handle.getChanges().size());
        for (StateChange change : handle.getChanges()) {
            dos.writeInt(change.getKeyGroup());
            dos.writeInt(change.getChange().length);
            dos.write(change.getChange());
        }
        writeStateHandleId(handle, dos);
    } else if (stateHandle instanceof ChangelogStateHandleStreamImpl) {
        ChangelogStateHandleStreamImpl handle = (ChangelogStateHandleStreamImpl) stateHandle;
        dos.writeByte(CHANGELOG_FILE_INCREMENT_HANDLE);
        dos.writeInt(handle.getKeyGroupRange().getStartKeyGroup());
        dos.writeInt(handle.getKeyGroupRange().getNumberOfKeyGroups());
        dos.writeInt(handle.getHandlesAndOffsets().size());
        for (Tuple2<StreamStateHandle, Long> streamHandleAndOffset : handle.getHandlesAndOffsets()) {
            dos.writeLong(streamHandleAndOffset.f1);
            serializeStreamStateHandle(streamHandleAndOffset.f0, dos);
        }
        dos.writeLong(handle.getStateSize());
        dos.writeLong(handle.getCheckpointedSize());
        writeStateHandleId(handle, dos);
    } else {
        throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
    }
}

Example 2 with StateChange

use of org.apache.flink.runtime.state.changelog.StateChange in project flink by apache.

the class BatchingStateChangeUploaderTest method getChanges.

private List<StateChangeSet> getChanges(int size) {
    byte[] change = new byte[size];
    random.nextBytes(change);
    return singletonList(new StateChangeSet(UUID.randomUUID(), SequenceNumber.of(0), singletonList(new StateChange(0, change))));
}

Example 3 with StateChange

use of org.apache.flink.runtime.state.changelog.StateChange in project flink by apache.

the class TestingStateChangeUploader method asBytesHandle.

private StreamStateHandle asBytesHandle(StateChangeSet changeSet) {
    byte[] bytes = new byte[(int) changeSet.getSize()];
    int offset = 0;
    for (StateChange change : changeSet.getChanges()) {
        for (int i = 0; i < change.getChange().length; i++) {
            bytes[offset++] = change.getChange()[i];
        }
    }
    return new ByteStreamStateHandle("", bytes);
}

Example 4 with StateChange

use of org.apache.flink.runtime.state.changelog.StateChange in project flink by apache.

the class FsStateChangelogWriter method append.

@Override
public void append(int keyGroup, byte[] value) throws IOException {
    LOG.trace("append to {}: keyGroup={} {} bytes", logId, keyGroup, value.length);
    checkState(!closed, "%s is closed", logId);
    activeChangeSet.add(new StateChange(keyGroup, value));
    activeChangeSetSize += value.length;
    if (activeChangeSetSize >= preEmptivePersistThresholdInBytes) {
        LOG.debug("pre-emptively flush {}Mb of appended changes to the common store", activeChangeSetSize / 1024 / 1024);
        persistInternal(notUploaded.isEmpty() ? activeSequenceNumber : notUploaded.firstKey());
    }
}

Example 5 with StateChange

use of org.apache.flink.runtime.state.changelog.StateChange in project flink by apache.

the class StateChangeFormat method read.

@Override
public CloseableIterator<StateChange> read(StreamStateHandle handle, long offset) throws IOException {
    FSDataInputStream stream = handle.openInputStream();
    DataInputViewStreamWrapper input = wrap(stream);
    if (stream.getPos() != offset) {
        LOG.debug("seek from {} to {}", stream.getPos(), offset);
        input.skipBytesToRead((int) offset);
    }
    return new CloseableIterator<StateChange>() {

        int numUnreadGroups = input.readInt();

        int numLeftInGroup = numUnreadGroups-- == 0 ? 0 : input.readInt();

        int keyGroup = numLeftInGroup == 0 ? 0 : input.readInt();

        @Override
        public boolean hasNext() {
            advance();
            return numLeftInGroup > 0;
        }

        private void advance() {
            if (numLeftInGroup == 0 && numUnreadGroups > 0) {
                numUnreadGroups--;
                try {
                    numLeftInGroup = input.readInt();
                    keyGroup = input.readInt();
                } catch (IOException e) {
                    ExceptionUtils.rethrow(e);
                }
            }
        }

        @Override
        public StateChange next() {
            advance();
            if (numLeftInGroup == 0) {
                throw new NoSuchElementException();
            }
            numLeftInGroup--;
            try {
                return readChange();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }

        private StateChange readChange() throws IOException {
            int size = input.readInt();
            byte[] bytes = new byte[size];
            IOUtils.readFully(input, bytes, 0, size);
            return new StateChange(keyGroup, bytes);
        }

        @Override
        public void close() throws Exception {
            LOG.trace("close {}", stream);
            stream.close();
        }
    };
}