Examples with StateHandleID org.apache.flink.runtime.state.StateHandleID used on opensource projects

Example 1 with StateHandleID

use of org.apache.flink.runtime.state.StateHandleID in project flink by apache.

the class CheckpointCoordinatorTest method ackCheckpoint.

private void ackCheckpoint(long checkpointId, CheckpointCoordinator coordinator, JobVertexID ackVertexID, ExecutionGraph graph, TestingStreamStateHandle metaState, TestingStreamStateHandle privateState, TestingStreamStateHandle sharedState) throws CheckpointException {
    Map<StateHandleID, StreamStateHandle> sharedStateMap = new HashMap<>(singletonMap(new StateHandleID("shared-state-key"), sharedState));
    Map<StateHandleID, StreamStateHandle> privateStateMap = new HashMap<>(singletonMap(new StateHandleID("private-state-key"), privateState));
    ExecutionJobVertex jobVertex = graph.getJobVertex(ackVertexID);
    OperatorID operatorID = jobVertex.getOperatorIDs().get(0).getGeneratedOperatorID();
    coordinator.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(graph.getJobID(), jobVertex.getTaskVertices()[0].getCurrentExecutionAttempt().getAttemptId(), checkpointId, new CheckpointMetrics(), new TaskStateSnapshot(singletonMap(operatorID, OperatorSubtaskState.builder().setManagedKeyedState(new IncrementalRemoteKeyedStateHandle(UUID.randomUUID(), KeyGroupRange.of(0, 9), checkpointId, sharedStateMap, privateStateMap, metaState)).build()))), "test");
}

Example 2 with StateHandleID

use of org.apache.flink.runtime.state.StateHandleID in project flink by apache.

the class MetadataV2V3SerializerBase method deserializeStreamStateHandleMap.

private static Map<StateHandleID, StreamStateHandle> deserializeStreamStateHandleMap(DataInputStream dis, @Nullable DeserializationContext context) throws IOException {
    final int size = dis.readInt();
    Map<StateHandleID, StreamStateHandle> result = new HashMap<>(size);
    for (int i = 0; i < size; ++i) {
        StateHandleID stateHandleID = new StateHandleID(dis.readUTF());
        StreamStateHandle stateHandle = deserializeStreamStateHandle(dis, context);
        result.put(stateHandleID, stateHandle);
    }
    return result;
}

Example 3 with StateHandleID

use of org.apache.flink.runtime.state.StateHandleID in project flink by apache.

the class CheckpointTestUtils method createRandomStateHandleMap.

public static Map<StateHandleID, StreamStateHandle> createRandomStateHandleMap(Random rnd) {
    final int size = rnd.nextInt(4);
    Map<StateHandleID, StreamStateHandle> result = new HashMap<>(size);
    for (int i = 0; i < size; ++i) {
        StateHandleID randomId = new StateHandleID(createRandomUUID(rnd).toString());
        StreamStateHandle stateHandle = createDummyStreamStateHandle(rnd, null);
        result.put(randomId, stateHandle);
    }
    return result;
}

Example 4 with StateHandleID

use of org.apache.flink.runtime.state.StateHandleID in project flink by apache.

the class RocksDBStateUploader method uploadFilesToCheckpointFs.

/**
 * Upload all the files to checkpoint fileSystem using specified number of threads.
 *
 * @param files The files will be uploaded to checkpoint filesystem.
 * @param checkpointStreamFactory The checkpoint streamFactory used to create outputstream.
 * @param stateScope
 * @throws Exception Thrown if can not upload all the files.
 */
public Map<StateHandleID, StreamStateHandle> uploadFilesToCheckpointFs(@Nonnull Map<StateHandleID, Path> files, CheckpointStreamFactory checkpointStreamFactory, CheckpointedStateScope stateScope, CloseableRegistry closeableRegistry) throws Exception {
    Map<StateHandleID, StreamStateHandle> handles = new HashMap<>();
    Map<StateHandleID, CompletableFuture<StreamStateHandle>> futures = createUploadFutures(files, checkpointStreamFactory, stateScope, closeableRegistry);
    try {
        FutureUtils.waitForAll(futures.values()).get();
        for (Map.Entry<StateHandleID, CompletableFuture<StreamStateHandle>> entry : futures.entrySet()) {
            handles.put(entry.getKey(), entry.getValue().get());
        }
    } catch (ExecutionException e) {
        Throwable throwable = ExceptionUtils.stripExecutionException(e);
        throwable = ExceptionUtils.stripException(throwable, RuntimeException.class);
        if (throwable instanceof IOException) {
            throw (IOException) throwable;
        } else {
            throw new FlinkRuntimeException("Failed to upload data for state handles.", e);
        }
    }
    return handles;
}

Example 5 with StateHandleID

use of org.apache.flink.runtime.state.StateHandleID in project flink by apache.

the class RocksDBStateDownloaderTest method testMultiThreadRestoreCorrectly.

/**
 * Tests that download files with multi-thread correctly.
 */
@Test
public void testMultiThreadRestoreCorrectly() throws Exception {
    Random random = new Random();
    int contentNum = 6;
    byte[][] contents = new byte[contentNum][];
    for (int i = 0; i < contentNum; ++i) {
        contents[i] = new byte[random.nextInt(100000) + 1];
        random.nextBytes(contents[i]);
    }
    List<StreamStateHandle> handles = new ArrayList<>(contentNum);
    for (int i = 0; i < contentNum; ++i) {
        handles.add(new ByteStreamStateHandle(String.format("state%d", i), contents[i]));
    }
    Map<StateHandleID, StreamStateHandle> sharedStates = new HashMap<>(contentNum);
    Map<StateHandleID, StreamStateHandle> privateStates = new HashMap<>(contentNum);
    for (int i = 0; i < contentNum; ++i) {
        sharedStates.put(new StateHandleID(String.format("sharedState%d", i)), handles.get(i));
        privateStates.put(new StateHandleID(String.format("privateState%d", i)), handles.get(i));
    }
    IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle = new IncrementalRemoteKeyedStateHandle(UUID.randomUUID(), KeyGroupRange.of(0, 1), 1, sharedStates, privateStates, handles.get(0));
    Path dstPath = temporaryFolder.newFolder().toPath();
    try (RocksDBStateDownloader rocksDBStateDownloader = new RocksDBStateDownloader(5)) {
        rocksDBStateDownloader.transferAllStateDataToDirectory(incrementalKeyedStateHandle, dstPath, new CloseableRegistry());
    }
    for (int i = 0; i < contentNum; ++i) {
        assertStateContentEqual(contents[i], dstPath.resolve(String.format("sharedState%d", i)));
    }
}