Examples with FSDataInputStream org.apache.flink.core.fs.FSDataInputStream used on opensource projects

Example 26 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class RocksDBStateDownloader method downloadDataForStateHandle.

/**
 * Copies the file from a single state handle to the given path.
 */
private void downloadDataForStateHandle(Path restoreFilePath, StreamStateHandle remoteFileHandle, CloseableRegistry closeableRegistry) throws IOException {
    FSDataInputStream inputStream = null;
    OutputStream outputStream = null;
    try {
        inputStream = remoteFileHandle.openInputStream();
        closeableRegistry.registerCloseable(inputStream);
        Files.createDirectories(restoreFilePath.getParent());
        outputStream = Files.newOutputStream(restoreFilePath);
        closeableRegistry.registerCloseable(outputStream);
        byte[] buffer = new byte[8 * 1024];
        while (true) {
            int numBytes = inputStream.read(buffer);
            if (numBytes == -1) {
                break;
            }
            outputStream.write(buffer, 0, numBytes);
        }
    } finally {
        if (closeableRegistry.unregisterCloseable(inputStream)) {
            inputStream.close();
        }
        if (closeableRegistry.unregisterCloseable(outputStream)) {
            outputStream.close();
        }
    }
}

Example 27 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class CheckpointStateOutputStreamTest method testWriteAndRead.

/**
 * Simple write and read test.
 */
@Test
public void testWriteAndRead() throws Exception {
    final FileSystem fs = FileSystem.getLocalFileSystem();
    final Path folder = baseFolder();
    final String fileName = "fooBarName";
    final Random rnd = new Random();
    final byte[] data = new byte[1694523];
    // write the data (mixed single byte writes and array writes)
    final FileStateHandle handle;
    try (FSDataOutputStream stream = createTestStream(fs, folder, fileName)) {
        for (int i = 0; i < data.length; ) {
            if (rnd.nextBoolean()) {
                stream.write(data[i++]);
            } else {
                int len = rnd.nextInt(Math.min(data.length - i, 32));
                stream.write(data, i, len);
                i += len;
            }
        }
        handle = closeAndGetResult(stream);
    }
    // (1) stream from handle must hold the contents
    try (FSDataInputStream in = handle.openInputStream()) {
        byte[] buffer = new byte[data.length];
        readFully(in, buffer);
        assertArrayEquals(data, buffer);
    }
    // (2) the pointer must point to a file with that contents
    try (FSDataInputStream in = fs.open(handle.getFilePath())) {
        byte[] buffer = new byte[data.length];
        readFully(in, buffer);
        assertArrayEquals(data, buffer);
    }
}

Example 28 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class ByteStreamStateHandleTest method testBulkRead.

@Test
public void testBulkRead() throws IOException {
    final byte[] data = { 34, 25, 22, 66 };
    final ByteStreamStateHandle handle = new ByteStreamStateHandle("name", data);
    final int targetLen = 8;
    for (int start = 0; start < data.length; start++) {
        for (int num = 0; num < targetLen; num++) {
            FSDataInputStream in = handle.openInputStream();
            in.seek(start);
            final byte[] target = new byte[targetLen];
            final int read = in.read(target, targetLen - num, num);
            assertEquals(Math.min(num, data.length - start), read);
            for (int i = 0; i < read; i++) {
                assertEquals(data[start + i], target[targetLen - num + i]);
            }
            int newPos = start + read;
            assertEquals(newPos, (int) in.getPos());
            assertEquals(newPos < data.length ? data[newPos] : -1, in.read());
        }
    }
}

Example 29 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class ByteStreamStateHandleTest method testStreamSeekAndPos.

@Test
public void testStreamSeekAndPos() throws IOException {
    final byte[] data = { 34, 25, 22, 66, 88, 54 };
    final ByteStreamStateHandle handle = new ByteStreamStateHandle("name", data);
    // read backwards, one byte at a time
    for (int i = data.length; i >= 0; i--) {
        FSDataInputStream in = handle.openInputStream();
        in.seek(i);
        assertEquals(i, (int) in.getPos());
        if (i < data.length) {
            assertEquals((int) data[i], in.read());
            assertEquals(i + 1, (int) in.getPos());
        } else {
            assertEquals(-1, in.read());
            assertEquals(i, (int) in.getPos());
        }
    }
    // reading past the end makes no difference
    FSDataInputStream in = handle.openInputStream();
    in.seek(data.length);
    // read multiple times, should not affect anything
    assertEquals(-1, in.read());
    assertEquals(-1, in.read());
    assertEquals(-1, in.read());
    assertEquals(data.length, (int) in.getPos());
}

Example 30 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class CheckpointStreamWithResultProviderTest method testCloseAndFinalizeCheckpointStreamResultPrimaryOnly.

@Test
public void testCloseAndFinalizeCheckpointStreamResultPrimaryOnly() throws Exception {
    CheckpointStreamFactory primaryFactory = createCheckpointStreamFactory();
    CheckpointStreamWithResultProvider resultProvider = CheckpointStreamWithResultProvider.createSimpleStream(CheckpointedStateScope.EXCLUSIVE, primaryFactory);
    SnapshotResult<StreamStateHandle> result = writeCheckpointTestData(resultProvider);
    Assert.assertNotNull(result.getJobManagerOwnedSnapshot());
    Assert.assertNull(result.getTaskLocalSnapshot());
    try (FSDataInputStream inputStream = result.getJobManagerOwnedSnapshot().openInputStream()) {
        Assert.assertEquals(0x42, inputStream.read());
        Assert.assertEquals(-1, inputStream.read());
    }
}