Examples with StreamStateHandle org.apache.flink.runtime.state.StreamStateHandle used on opensource projects

Example 76 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class InterruptSensitiveRestoreTest method testRestoreWithInterrupt.

private void testRestoreWithInterrupt(int mode) throws Exception {
    IN_RESTORE_LATCH.reset();
    Configuration taskConfig = new Configuration();
    StreamConfig cfg = new StreamConfig(taskConfig);
    cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    switch(mode) {
        case OPERATOR_MANAGED:
        case OPERATOR_RAW:
        case KEYED_MANAGED:
        case KEYED_RAW:
            cfg.setStateKeySerializer(IntSerializer.INSTANCE);
            cfg.setStreamOperator(new StreamSource<>(new TestSource(mode)));
            break;
        default:
            throw new IllegalArgumentException();
    }
    StreamStateHandle lockingHandle = new InterruptLockingStateHandle();
    Task task = createTask(cfg, taskConfig, lockingHandle, mode);
    // start the task and wait until it is in "restore"
    task.startTaskThread();
    IN_RESTORE_LATCH.await();
    // trigger cancellation and signal to continue
    task.cancelExecution();
    task.getExecutingThread().join(30000);
    if (task.getExecutionState() == ExecutionState.CANCELING) {
        fail("Task is stuck and not canceling");
    }
    assertEquals(ExecutionState.CANCELED, task.getExecutionState());
    assertNull(task.getFailureCause());
}

Example 77 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class FsCheckpointStateOutputStreamTest method runTest.

private void runTest(int numBytes, int bufferSize, int threshold, boolean expectFile) throws Exception {
    CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), bufferSize, threshold, relativePaths);
    Random rnd = new Random();
    byte[] original = new byte[numBytes];
    byte[] bytes = new byte[original.length];
    rnd.nextBytes(original);
    System.arraycopy(original, 0, bytes, 0, original.length);
    // the test writes a mixture of writing individual bytes and byte arrays
    int pos = 0;
    while (pos < bytes.length) {
        boolean single = rnd.nextBoolean();
        if (single) {
            stream.write(bytes[pos++]);
        } else {
            int num = rnd.nextBoolean() ? (bytes.length - pos) : rnd.nextInt(bytes.length - pos);
            stream.write(bytes, pos, num);
            pos += num;
        }
    }
    StreamStateHandle handle = stream.closeAndGetHandle();
    if (expectFile) {
        assertTrue(handle instanceof FileStateHandle);
    } else {
        assertTrue(handle instanceof ByteStreamStateHandle);
    }
    // make sure the writing process did not alter the original byte array
    assertArrayEquals(original, bytes);
    try (InputStream inStream = handle.openInputStream()) {
        byte[] validation = new byte[bytes.length];
        DataInputStream dataInputStream = new DataInputStream(inStream);
        dataInputStream.readFully(validation);
        assertArrayEquals(bytes, validation);
    }
    handle.discardState();
}

Example 78 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class RocksDBStateUploaderTest method testMultiThreadUploadCorrectly.

/**
 * Test that upload files with multi-thread correctly.
 */
@Test
public void testMultiThreadUploadCorrectly() throws Exception {
    File checkpointPrivateFolder = temporaryFolder.newFolder("private");
    org.apache.flink.core.fs.Path checkpointPrivateDirectory = org.apache.flink.core.fs.Path.fromLocalFile(checkpointPrivateFolder);
    File checkpointSharedFolder = temporaryFolder.newFolder("shared");
    org.apache.flink.core.fs.Path checkpointSharedDirectory = org.apache.flink.core.fs.Path.fromLocalFile(checkpointSharedFolder);
    FileSystem fileSystem = checkpointPrivateDirectory.getFileSystem();
    int fileStateSizeThreshold = 1024;
    int writeBufferSize = 4096;
    FsCheckpointStreamFactory checkpointStreamFactory = new FsCheckpointStreamFactory(fileSystem, checkpointPrivateDirectory, checkpointSharedDirectory, fileStateSizeThreshold, writeBufferSize);
    String localFolder = "local";
    temporaryFolder.newFolder(localFolder);
    int sstFileCount = 6;
    Map<StateHandleID, Path> sstFilePaths = generateRandomSstFiles(localFolder, sstFileCount, fileStateSizeThreshold);
    try (RocksDBStateUploader rocksDBStateUploader = new RocksDBStateUploader(5)) {
        Map<StateHandleID, StreamStateHandle> sstFiles = rocksDBStateUploader.uploadFilesToCheckpointFs(sstFilePaths, checkpointStreamFactory, CheckpointedStateScope.SHARED, new CloseableRegistry());
        for (Map.Entry<StateHandleID, Path> entry : sstFilePaths.entrySet()) {
            assertStateContentEqual(entry.getValue(), sstFiles.get(entry.getKey()).openInputStream());
        }
    }
}

Example 79 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class RocksIncrementalSnapshotStrategyTest method testCheckpointIsIncremental.

// Verify the next checkpoint is still incremental after a savepoint completed.
@Test
public void testCheckpointIsIncremental() throws Exception {
    try (CloseableRegistry closeableRegistry = new CloseableRegistry();
        RocksIncrementalSnapshotStrategy checkpointSnapshotStrategy = createSnapshotStrategy(closeableRegistry)) {
        FsCheckpointStreamFactory checkpointStreamFactory = createFsCheckpointStreamFactory();
        // make and notify checkpoint with id 1
        snapshot(1L, checkpointSnapshotStrategy, checkpointStreamFactory, closeableRegistry);
        checkpointSnapshotStrategy.notifyCheckpointComplete(1L);
        // notify savepoint with id 2
        checkpointSnapshotStrategy.notifyCheckpointComplete(2L);
        // make checkpoint with id 3
        IncrementalRemoteKeyedStateHandle incrementalRemoteKeyedStateHandle3 = snapshot(3L, checkpointSnapshotStrategy, checkpointStreamFactory, closeableRegistry);
        // If 3rd checkpoint's placeholderStateHandleCount > 0,it means 3rd checkpoint is
        // incremental.
        Map<StateHandleID, StreamStateHandle> sharedState3 = incrementalRemoteKeyedStateHandle3.getSharedState();
        long placeholderStateHandleCount = sharedState3.entrySet().stream().filter(e -> e.getValue() instanceof PlaceholderStreamStateHandle).count();
        Assert.assertTrue(placeholderStateHandleCount > 0);
    }
}

Example 80 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class EmbeddedRocksDBStateBackendTest method testSharedIncrementalStateDeRegistration.

@Test
public void testSharedIncrementalStateDeRegistration() throws Exception {
    if (enableIncrementalCheckpointing) {
        CheckpointableKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
        try {
            ValueStateDescriptor<String> kvId = new ValueStateDescriptor<>("id", String.class, null);
            kvId.initializeSerializerUnlessSet(new ExecutionConfig());
            ValueState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
            Queue<IncrementalRemoteKeyedStateHandle> previousStateHandles = new LinkedList<>();
            SharedStateRegistry sharedStateRegistry = spy(new SharedStateRegistryImpl());
            for (int checkpointId = 0; checkpointId < 3; ++checkpointId) {
                reset(sharedStateRegistry);
                backend.setCurrentKey(checkpointId);
                state.update("Hello-" + checkpointId);
                RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = backend.snapshot(checkpointId, checkpointId, createStreamFactory(), CheckpointOptions.forCheckpointWithDefaultLocation());
                snapshot.run();
                SnapshotResult<KeyedStateHandle> snapshotResult = snapshot.get();
                IncrementalRemoteKeyedStateHandle stateHandle = (IncrementalRemoteKeyedStateHandle) snapshotResult.getJobManagerOwnedSnapshot();
                Map<StateHandleID, StreamStateHandle> sharedState = new HashMap<>(stateHandle.getSharedState());
                stateHandle.registerSharedStates(sharedStateRegistry, checkpointId);
                for (Map.Entry<StateHandleID, StreamStateHandle> e : sharedState.entrySet()) {
                    verify(sharedStateRegistry).registerReference(stateHandle.createSharedStateRegistryKeyFromFileName(e.getKey()), e.getValue(), checkpointId);
                }
                previousStateHandles.add(stateHandle);
                ((CheckpointListener) backend).notifyCheckpointComplete(checkpointId);
                if (previousStateHandles.size() > 1) {
                    previousStateHandles.remove().discardState();
                }
            }
            while (!previousStateHandles.isEmpty()) {
                reset(sharedStateRegistry);
                previousStateHandles.remove().discardState();
            }
        } finally {
            IOUtils.closeQuietly(backend);
            backend.dispose();
        }
    }
}