Examples with StateHandleID org.apache.flink.runtime.state.StateHandleID used on opensource projects

Example 11 with StateHandleID

use of org.apache.flink.runtime.state.StateHandleID in project flink by apache.

the class MetadataV2V3SerializerBase method deserializeKeyedStateHandle.

@VisibleForTesting
@Nullable
static KeyedStateHandle deserializeKeyedStateHandle(DataInputStream dis, @Nullable DeserializationContext context) throws IOException {
    final int type = dis.readByte();
    if (NULL_HANDLE == type) {
        return null;
    } else if (KEY_GROUPS_HANDLE == type || KEY_GROUPS_HANDLE_V2 == type || SAVEPOINT_KEY_GROUPS_HANDLE == type) {
        int startKeyGroup = dis.readInt();
        int numKeyGroups = dis.readInt();
        KeyGroupRange keyGroupRange = KeyGroupRange.of(startKeyGroup, startKeyGroup + numKeyGroups - 1);
        long[] offsets = new long[numKeyGroups];
        for (int i = 0; i < numKeyGroups; ++i) {
            offsets[i] = dis.readLong();
        }
        KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(keyGroupRange, offsets);
        StreamStateHandle stateHandle = deserializeStreamStateHandle(dis, context);
        if (SAVEPOINT_KEY_GROUPS_HANDLE == type) {
            return new KeyGroupsSavepointStateHandle(keyGroupRangeOffsets, stateHandle);
        } else {
            StateHandleID stateHandleID = KEY_GROUPS_HANDLE_V2 == type ? new StateHandleID(dis.readUTF()) : StateHandleID.randomStateHandleId();
            return KeyGroupsStateHandle.restore(keyGroupRangeOffsets, stateHandle, stateHandleID);
        }
    } else if (INCREMENTAL_KEY_GROUPS_HANDLE == type || INCREMENTAL_KEY_GROUPS_HANDLE_V2 == type) {
        return deserializeIncrementalStateHandle(dis, context, type);
    } else if (CHANGELOG_HANDLE == type) {
        int startKeyGroup = dis.readInt();
        int numKeyGroups = dis.readInt();
        KeyGroupRange keyGroupRange = KeyGroupRange.of(startKeyGroup, startKeyGroup + numKeyGroups - 1);
        long checkpointedSize = dis.readLong();
        int baseSize = dis.readInt();
        List<KeyedStateHandle> base = new ArrayList<>(baseSize);
        for (int i = 0; i < baseSize; i++) {
            KeyedStateHandle handle = deserializeKeyedStateHandle(dis, context);
            if (handle != null) {
                base.add(handle);
            } else {
                LOG.warn("Unexpected null keyed state handle of materialized part when deserializing changelog state-backend handle");
            }
        }
        int deltaSize = dis.readInt();
        List<ChangelogStateHandle> delta = new ArrayList<>(deltaSize);
        for (int i = 0; i < deltaSize; i++) {
            delta.add((ChangelogStateHandle) deserializeKeyedStateHandle(dis, context));
        }
        long materializationID = dis.readLong();
        StateHandleID stateHandleId = new StateHandleID(dis.readUTF());
        return ChangelogStateBackendHandleImpl.restore(base, delta, keyGroupRange, materializationID, checkpointedSize, stateHandleId);
    } else if (CHANGELOG_BYTE_INCREMENT_HANDLE == type) {
        int start = dis.readInt();
        int numKeyGroups = dis.readInt();
        KeyGroupRange keyGroupRange = KeyGroupRange.of(start, start + numKeyGroups - 1);
        long from = dis.readLong();
        long to = dis.readLong();
        int size = dis.readInt();
        List<StateChange> changes = new ArrayList<>(size);
        for (int i = 0; i < size; i++) {
            int keyGroup = dis.readInt();
            int bytesSize = dis.readInt();
            byte[] bytes = new byte[bytesSize];
            IOUtils.readFully(dis, bytes, 0, bytesSize);
            changes.add(new StateChange(keyGroup, bytes));
        }
        StateHandleID stateHandleId = new StateHandleID(dis.readUTF());
        return InMemoryChangelogStateHandle.restore(changes, SequenceNumber.of(from), SequenceNumber.of(to), keyGroupRange, stateHandleId);
    } else if (CHANGELOG_FILE_INCREMENT_HANDLE == type) {
        int start = dis.readInt();
        int numKeyGroups = dis.readInt();
        KeyGroupRange keyGroupRange = KeyGroupRange.of(start, start + numKeyGroups - 1);
        int numHandles = dis.readInt();
        List<Tuple2<StreamStateHandle, Long>> streamHandleAndOffset = new ArrayList<>(numHandles);
        for (int i = 0; i < numHandles; i++) {
            long o = dis.readLong();
            StreamStateHandle h = deserializeStreamStateHandle(dis, context);
            streamHandleAndOffset.add(Tuple2.of(h, o));
        }
        long size = dis.readLong();
        long checkpointedSize = dis.readLong();
        StateHandleID stateHandleId = new StateHandleID(dis.readUTF());
        return ChangelogStateHandleStreamImpl.restore(streamHandleAndOffset, keyGroupRange, size, checkpointedSize, stateHandleId);
    } else {
        throw new IllegalStateException("Reading invalid KeyedStateHandle, type: " + type);
    }
}

Example 12 with StateHandleID

use of org.apache.flink.runtime.state.StateHandleID in project flink by apache.

the class RocksDBStateDownloader method createDownloadRunnables.

private List<Runnable> createDownloadRunnables(Map<StateHandleID, StreamStateHandle> stateHandleMap, Path restoreInstancePath, CloseableRegistry closeableRegistry) {
    List<Runnable> runnables = new ArrayList<>(stateHandleMap.size());
    for (Map.Entry<StateHandleID, StreamStateHandle> entry : stateHandleMap.entrySet()) {
        StateHandleID stateHandleID = entry.getKey();
        StreamStateHandle remoteFileHandle = entry.getValue();
        Path path = restoreInstancePath.resolve(stateHandleID.toString());
        runnables.add(ThrowingRunnable.unchecked(() -> downloadDataForStateHandle(path, remoteFileHandle, closeableRegistry)));
    }
    return runnables;
}

Example 13 with StateHandleID

use of org.apache.flink.runtime.state.StateHandleID in project flink by apache.

the class RocksDBStateUploaderTest method testMultiThreadUploadCorrectly.

/**
 * Test that upload files with multi-thread correctly.
 */
@Test
public void testMultiThreadUploadCorrectly() throws Exception {
    File checkpointPrivateFolder = temporaryFolder.newFolder("private");
    org.apache.flink.core.fs.Path checkpointPrivateDirectory = org.apache.flink.core.fs.Path.fromLocalFile(checkpointPrivateFolder);
    File checkpointSharedFolder = temporaryFolder.newFolder("shared");
    org.apache.flink.core.fs.Path checkpointSharedDirectory = org.apache.flink.core.fs.Path.fromLocalFile(checkpointSharedFolder);
    FileSystem fileSystem = checkpointPrivateDirectory.getFileSystem();
    int fileStateSizeThreshold = 1024;
    int writeBufferSize = 4096;
    FsCheckpointStreamFactory checkpointStreamFactory = new FsCheckpointStreamFactory(fileSystem, checkpointPrivateDirectory, checkpointSharedDirectory, fileStateSizeThreshold, writeBufferSize);
    String localFolder = "local";
    temporaryFolder.newFolder(localFolder);
    int sstFileCount = 6;
    Map<StateHandleID, Path> sstFilePaths = generateRandomSstFiles(localFolder, sstFileCount, fileStateSizeThreshold);
    try (RocksDBStateUploader rocksDBStateUploader = new RocksDBStateUploader(5)) {
        Map<StateHandleID, StreamStateHandle> sstFiles = rocksDBStateUploader.uploadFilesToCheckpointFs(sstFilePaths, checkpointStreamFactory, CheckpointedStateScope.SHARED, new CloseableRegistry());
        for (Map.Entry<StateHandleID, Path> entry : sstFilePaths.entrySet()) {
            assertStateContentEqual(entry.getValue(), sstFiles.get(entry.getKey()).openInputStream());
        }
    }
}

Example 14 with StateHandleID

use of org.apache.flink.runtime.state.StateHandleID in project flink by apache.

the class RocksDBStateUploaderTest method generateRandomSstFiles.

private Map<StateHandleID, Path> generateRandomSstFiles(String localFolder, int sstFileCount, int fileStateSizeThreshold) throws IOException {
    ThreadLocalRandom random = ThreadLocalRandom.current();
    Map<StateHandleID, Path> sstFilePaths = new HashMap<>(sstFileCount);
    for (int i = 0; i < sstFileCount; ++i) {
        File file = temporaryFolder.newFile(String.format("%s/%d.sst", localFolder, i));
        generateRandomFileContent(file.getPath(), random.nextInt(1_000_000) + fileStateSizeThreshold);
        sstFilePaths.put(new StateHandleID(String.valueOf(i)), file.toPath());
    }
    return sstFilePaths;
}

Example 15 with StateHandleID

use of org.apache.flink.runtime.state.StateHandleID in project flink by apache.

the class RocksIncrementalSnapshotStrategyTest method testCheckpointIsIncremental.

// Verify the next checkpoint is still incremental after a savepoint completed.
@Test
public void testCheckpointIsIncremental() throws Exception {
    try (CloseableRegistry closeableRegistry = new CloseableRegistry();
        RocksIncrementalSnapshotStrategy checkpointSnapshotStrategy = createSnapshotStrategy(closeableRegistry)) {
        FsCheckpointStreamFactory checkpointStreamFactory = createFsCheckpointStreamFactory();
        // make and notify checkpoint with id 1
        snapshot(1L, checkpointSnapshotStrategy, checkpointStreamFactory, closeableRegistry);
        checkpointSnapshotStrategy.notifyCheckpointComplete(1L);
        // notify savepoint with id 2
        checkpointSnapshotStrategy.notifyCheckpointComplete(2L);
        // make checkpoint with id 3
        IncrementalRemoteKeyedStateHandle incrementalRemoteKeyedStateHandle3 = snapshot(3L, checkpointSnapshotStrategy, checkpointStreamFactory, closeableRegistry);
        // If 3rd checkpoint's placeholderStateHandleCount > 0,it means 3rd checkpoint is
        // incremental.
        Map<StateHandleID, StreamStateHandle> sharedState3 = incrementalRemoteKeyedStateHandle3.getSharedState();
        long placeholderStateHandleCount = sharedState3.entrySet().stream().filter(e -> e.getValue() instanceof PlaceholderStreamStateHandle).count();
        Assert.assertTrue(placeholderStateHandleCount > 0);
    }
}