Examples with ChangelogStateHandleStreamImpl org.apache.flink.runtime.state.changelog.ChangelogStateHandleStreamImpl used on opensource projects

Example 1 with ChangelogStateHandleStreamImpl

use of org.apache.flink.runtime.state.changelog.ChangelogStateHandleStreamImpl in project flink by apache.

the class MetadataV2V3SerializerBase method serializeKeyedStateHandle.

// ------------------------------------------------------------------------
// keyed state
// ------------------------------------------------------------------------
@VisibleForTesting
static void serializeKeyedStateHandle(KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {
    if (stateHandle == null) {
        dos.writeByte(NULL_HANDLE);
    } else if (stateHandle instanceof KeyGroupsStateHandle) {
        KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;
        if (stateHandle instanceof KeyGroupsSavepointStateHandle) {
            dos.writeByte(SAVEPOINT_KEY_GROUPS_HANDLE);
        } else {
            dos.writeByte(KEY_GROUPS_HANDLE_V2);
        }
        dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
        dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
        for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
            dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
        }
        serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
        // savepoint state handle would not need to persist state handle id out.
        if (!(stateHandle instanceof KeyGroupsSavepointStateHandle)) {
            writeStateHandleId(stateHandle, dos);
        }
    } else if (stateHandle instanceof IncrementalRemoteKeyedStateHandle) {
        IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle = (IncrementalRemoteKeyedStateHandle) stateHandle;
        dos.writeByte(INCREMENTAL_KEY_GROUPS_HANDLE_V2);
        dos.writeLong(incrementalKeyedStateHandle.getCheckpointId());
        dos.writeUTF(String.valueOf(incrementalKeyedStateHandle.getBackendIdentifier()));
        dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getStartKeyGroup());
        dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
        dos.writeLong(incrementalKeyedStateHandle.getCheckpointedSize());
        serializeStreamStateHandle(incrementalKeyedStateHandle.getMetaStateHandle(), dos);
        serializeStreamStateHandleMap(incrementalKeyedStateHandle.getSharedState(), dos);
        serializeStreamStateHandleMap(incrementalKeyedStateHandle.getPrivateState(), dos);
        writeStateHandleId(incrementalKeyedStateHandle, dos);
    } else if (stateHandle instanceof ChangelogStateBackendHandle) {
        ChangelogStateBackendHandle handle = (ChangelogStateBackendHandle) stateHandle;
        dos.writeByte(CHANGELOG_HANDLE);
        dos.writeInt(handle.getKeyGroupRange().getStartKeyGroup());
        dos.writeInt(handle.getKeyGroupRange().getNumberOfKeyGroups());
        dos.writeLong(handle.getCheckpointedSize());
        dos.writeInt(handle.getMaterializedStateHandles().size());
        for (KeyedStateHandle keyedStateHandle : handle.getMaterializedStateHandles()) {
            serializeKeyedStateHandle(keyedStateHandle, dos);
        }
        dos.writeInt(handle.getNonMaterializedStateHandles().size());
        for (KeyedStateHandle k : handle.getNonMaterializedStateHandles()) {
            serializeKeyedStateHandle(k, dos);
        }
        dos.writeLong(handle.getMaterializationID());
        writeStateHandleId(handle, dos);
    } else if (stateHandle instanceof InMemoryChangelogStateHandle) {
        InMemoryChangelogStateHandle handle = (InMemoryChangelogStateHandle) stateHandle;
        dos.writeByte(CHANGELOG_BYTE_INCREMENT_HANDLE);
        dos.writeInt(handle.getKeyGroupRange().getStartKeyGroup());
        dos.writeInt(handle.getKeyGroupRange().getNumberOfKeyGroups());
        dos.writeLong(handle.getFrom());
        dos.writeLong(handle.getTo());
        dos.writeInt(handle.getChanges().size());
        for (StateChange change : handle.getChanges()) {
            dos.writeInt(change.getKeyGroup());
            dos.writeInt(change.getChange().length);
            dos.write(change.getChange());
        }
        writeStateHandleId(handle, dos);
    } else if (stateHandle instanceof ChangelogStateHandleStreamImpl) {
        ChangelogStateHandleStreamImpl handle = (ChangelogStateHandleStreamImpl) stateHandle;
        dos.writeByte(CHANGELOG_FILE_INCREMENT_HANDLE);
        dos.writeInt(handle.getKeyGroupRange().getStartKeyGroup());
        dos.writeInt(handle.getKeyGroupRange().getNumberOfKeyGroups());
        dos.writeInt(handle.getHandlesAndOffsets().size());
        for (Tuple2<StreamStateHandle, Long> streamHandleAndOffset : handle.getHandlesAndOffsets()) {
            dos.writeLong(streamHandleAndOffset.f1);
            serializeStreamStateHandle(streamHandleAndOffset.f0, dos);
        }
        dos.writeLong(handle.getStateSize());
        dos.writeLong(handle.getCheckpointedSize());
        writeStateHandleId(handle, dos);
    } else {
        throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
    }
}

Example 2 with ChangelogStateHandleStreamImpl

use of org.apache.flink.runtime.state.changelog.ChangelogStateHandleStreamImpl in project flink by apache.

the class FsStateChangelogWriterTest method testPersistFailure.

@Test(expected = IOException.class)
public void testPersistFailure() throws Exception {
    withWriter((writer, uploader) -> {
        byte[] bytes = getBytes();
        SequenceNumber sqn = append(writer, bytes);
        CompletableFuture<ChangelogStateHandleStreamImpl> future = writer.persist(sqn);
        uploader.failUpload(new RuntimeException("test"));
        try {
            future.get();
        } catch (ExecutionException e) {
            rethrowIOException(e.getCause());
        }
    });
}

Example 3 with ChangelogStateHandleStreamImpl

use of org.apache.flink.runtime.state.changelog.ChangelogStateHandleStreamImpl in project flink by apache.

the class FsStateChangelogWriterTest method testPersistNonFailedChanges.

@Test
public void testPersistNonFailedChanges() throws Exception {
    withWriter((writer, uploader) -> {
        byte[] bytes = getBytes();
        SequenceNumber sqn1 = append(writer, bytes);
        // future result ignored
        writer.persist(sqn1);
        uploader.failUpload(new RuntimeException("test"));
        uploader.reset();
        SequenceNumber sqn2 = append(writer, bytes);
        CompletableFuture<ChangelogStateHandleStreamImpl> future = writer.persist(sqn2);
        uploader.completeUpload();
        future.get();
    });
}

Example 4 with ChangelogStateHandleStreamImpl

use of org.apache.flink.runtime.state.changelog.ChangelogStateHandleStreamImpl in project flink by apache.

the class FsStateChangelogWriter method persistInternal.

private CompletableFuture<ChangelogStateHandleStreamImpl> persistInternal(SequenceNumber from) throws IOException {
    synchronized (lock) {
        ensureCanPersist(from);
        rollover();
        Map<SequenceNumber, StateChangeSet> toUpload = drainTailMap(notUploaded, from);
        NavigableMap<SequenceNumber, UploadResult> readyToReturn = uploaded.tailMap(from, true);
        LOG.debug("collected readyToReturn: {}, toUpload: {}", readyToReturn, toUpload);
        SequenceNumberRange range = SequenceNumberRange.generic(from, activeSequenceNumber);
        if (range.size() == readyToReturn.size()) {
            checkState(toUpload.isEmpty());
            return completedFuture(buildHandle(keyGroupRange, readyToReturn, 0L));
        } else {
            CompletableFuture<ChangelogStateHandleStreamImpl> future = new CompletableFuture<>();
            uploadCompletionListeners.add(new UploadCompletionListener(keyGroupRange, range, readyToReturn, future));
            if (!toUpload.isEmpty()) {
                uploader.upload(new UploadTask(toUpload.values(), this::handleUploadSuccess, this::handleUploadFailure));
            }
            return future;
        }
    }
}