Examples with CheckpointStateOutputStream org.apache.flink.runtime.state.CheckpointStateOutputStream used on opensource projects

Example 16 with CheckpointStateOutputStream

use of org.apache.flink.runtime.state.CheckpointStateOutputStream in project flink by apache.

the class FsCheckpointStateOutputStreamTest method runTest.

private void runTest(int numBytes, int bufferSize, int threshold, boolean expectFile) throws Exception {
    CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), bufferSize, threshold, relativePaths);
    Random rnd = new Random();
    byte[] original = new byte[numBytes];
    byte[] bytes = new byte[original.length];
    rnd.nextBytes(original);
    System.arraycopy(original, 0, bytes, 0, original.length);
    // the test writes a mixture of writing individual bytes and byte arrays
    int pos = 0;
    while (pos < bytes.length) {
        boolean single = rnd.nextBoolean();
        if (single) {
            stream.write(bytes[pos++]);
        } else {
            int num = rnd.nextBoolean() ? (bytes.length - pos) : rnd.nextInt(bytes.length - pos);
            stream.write(bytes, pos, num);
            pos += num;
        }
    }
    StreamStateHandle handle = stream.closeAndGetHandle();
    if (expectFile) {
        assertTrue(handle instanceof FileStateHandle);
    } else {
        assertTrue(handle instanceof ByteStreamStateHandle);
    }
    // make sure the writing process did not alter the original byte array
    assertArrayEquals(original, bytes);
    try (InputStream inStream = handle.openInputStream()) {
        byte[] validation = new byte[bytes.length];
        DataInputStream dataInputStream = new DataInputStream(inStream);
        dataInputStream.readFully(validation);
        assertArrayEquals(bytes, validation);
    }
    handle.discardState();
}

Example 17 with CheckpointStateOutputStream

use of org.apache.flink.runtime.state.CheckpointStateOutputStream in project flink by apache.

the class FsCheckpointStateOutputStreamTest method testGetPos.

@Test
public void testGetPos() throws Exception {
    CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 31, 17, relativePaths);
    for (int i = 0; i < 64; ++i) {
        Assert.assertEquals(i, stream.getPos());
        stream.write(0x42);
    }
    stream.closeAndGetHandle();
    // ----------------------------------------------------
    stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 31, 17, relativePaths);
    byte[] data = "testme!".getBytes(ConfigConstants.DEFAULT_CHARSET);
    for (int i = 0; i < 7; ++i) {
        Assert.assertEquals(i * (1 + data.length), stream.getPos());
        stream.write(0x42);
        stream.write(data);
    }
    stream.closeAndGetHandle();
}

Example 18 with CheckpointStateOutputStream

use of org.apache.flink.runtime.state.CheckpointStateOutputStream in project flink by apache.

the class RocksDBAsyncSnapshotTest method testCleanupOfSnapshotsInFailureCase.

/**
 * Test that the snapshot files are cleaned up in case of a failure during the snapshot
 * procedure.
 */
@Test
public void testCleanupOfSnapshotsInFailureCase() throws Exception {
    long checkpointId = 1L;
    long timestamp = 42L;
    MockEnvironment env = MockEnvironment.builder().build();
    final IOException testException = new IOException("Test exception");
    CheckpointStateOutputStream outputStream = spy(new FailingStream(testException));
    RocksDBStateBackend backend = new RocksDBStateBackend((StateBackend) new MemoryStateBackend());
    backend.setDbStoragePath(temporaryFolder.newFolder().toURI().toString());
    AbstractKeyedStateBackend<Void> keyedStateBackend = backend.createKeyedStateBackend(env, new JobID(), "test operator", VoidSerializer.INSTANCE, 1, new KeyGroupRange(0, 0), null, TtlTimeProvider.DEFAULT, new UnregisteredMetricsGroup(), Collections.emptyList(), new CloseableRegistry());
    try {
        // register a state so that the state backend has to checkpoint something
        keyedStateBackend.getPartitionedState("namespace", StringSerializer.INSTANCE, new ValueStateDescriptor<>("foobar", String.class));
        RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshotFuture = keyedStateBackend.snapshot(checkpointId, timestamp, new TestCheckpointStreamFactory(() -> outputStream), CheckpointOptions.forCheckpointWithDefaultLocation());
        try {
            FutureUtils.runIfNotDoneAndGet(snapshotFuture);
            fail("Expected an exception to be thrown here.");
        } catch (ExecutionException e) {
            Assert.assertEquals(testException, e.getCause());
        }
        verify(outputStream).close();
    } finally {
        IOUtils.closeQuietly(keyedStateBackend);
        keyedStateBackend.dispose();
        IOUtils.closeQuietly(env);
    }
}