Examples with FSDataInputStream org.apache.flink.core.fs.FSDataInputStream used on opensource projects

Example 51 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class DuplicatingCheckpointOutputStreamTest method testUnalignedStreamsException.

/**
 * Tests that in case of unaligned stream positions, the secondary stream is closed and the
 * primary still works. This is important because some code may rely on seeking to stream
 * offsets in the created state files and if the streams are not aligned this code could fail.
 */
@Test
public void testUnalignedStreamsException() throws IOException {
    int streamCapacity = 1024 * 1024;
    TestMemoryCheckpointOutputStream primaryStream = new TestMemoryCheckpointOutputStream(streamCapacity);
    TestMemoryCheckpointOutputStream secondaryStream = new TestMemoryCheckpointOutputStream(streamCapacity);
    primaryStream.write(42);
    DuplicatingCheckpointOutputStream stream = new DuplicatingCheckpointOutputStream(primaryStream, secondaryStream);
    Assert.assertNotNull(stream.getSecondaryStreamException());
    Assert.assertTrue(secondaryStream.isClosed());
    stream.write(23);
    try {
        stream.closeAndGetSecondaryHandle();
        Assert.fail();
    } catch (IOException ignore) {
        Assert.assertEquals(ignore.getCause(), stream.getSecondaryStreamException());
    }
    StreamStateHandle primaryHandle = stream.closeAndGetPrimaryHandle();
    try (FSDataInputStream inputStream = primaryHandle.openInputStream()) {
        Assert.assertEquals(42, inputStream.read());
        Assert.assertEquals(23, inputStream.read());
        Assert.assertEquals(-1, inputStream.read());
    }
}

Example 52 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class ByteStreamStateHandleTest method testStreamSeekOutOfBounds.

@Test
public void testStreamSeekOutOfBounds() throws IOException {
    final int len = 10;
    final ByteStreamStateHandle handle = new ByteStreamStateHandle("name", new byte[len]);
    // check negative offset
    FSDataInputStream in = handle.openInputStream();
    try {
        in.seek(-2);
        fail("should fail with an exception");
    } catch (IOException e) {
    // expected
    }
    // check integer overflow
    in = handle.openInputStream();
    try {
        in.seek(len + 1);
        fail("should fail with an exception");
    } catch (IOException e) {
    // expected
    }
    // check integer overflow
    in = handle.openInputStream();
    try {
        in.seek(((long) Integer.MAX_VALUE) + 100L);
        fail("should fail with an exception");
    } catch (IOException e) {
    // expected
    }
}

Example 53 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class CheckpointStateOutputStreamTest method testEmptyState.

// ------------------------------------------------------------------------
// Tests
// ------------------------------------------------------------------------
/**
 * Validates that even empty streams create a file and a file state handle.
 */
@Test
public void testEmptyState() throws Exception {
    final FileSystem fs = FileSystem.getLocalFileSystem();
    final Path folder = baseFolder();
    final String fileName = "myFileName";
    final Path filePath = new Path(folder, fileName);
    final FileStateHandle handle;
    try (FSDataOutputStream stream = createTestStream(fs, folder, fileName)) {
        handle = closeAndGetResult(stream);
    }
    // must have created a handle
    assertNotNull(handle);
    assertEquals(filePath, handle.getFilePath());
    // the pointer path should exist as a directory
    assertTrue(fs.exists(handle.getFilePath()));
    assertFalse(fs.getFileStatus(filePath).isDir());
    // the contents should be empty
    try (FSDataInputStream in = handle.openInputStream()) {
        assertEquals(-1, in.read());
    }
}

Example 54 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class ByteStreamStateHandleTest method testBulkReadINdexOutOfBounds.

@SuppressWarnings("ResultOfMethodCallIgnored")
@Test
public void testBulkReadINdexOutOfBounds() throws IOException {
    final ByteStreamStateHandle handle = new ByteStreamStateHandle("name", new byte[10]);
    // check negative offset
    FSDataInputStream in = handle.openInputStream();
    try {
        in.read(new byte[10], -1, 5);
        fail("should fail with an exception");
    } catch (IndexOutOfBoundsException e) {
    // expected
    }
    // check offset overflow
    in = handle.openInputStream();
    try {
        in.read(new byte[10], 10, 5);
        fail("should fail with an exception");
    } catch (IndexOutOfBoundsException e) {
    // expected
    }
    // check negative length
    in = handle.openInputStream();
    try {
        in.read(new byte[10], 0, -2);
        fail("should fail with an exception");
    } catch (IndexOutOfBoundsException e) {
    // expected
    }
    // check length too large
    in = handle.openInputStream();
    try {
        in.read(new byte[10], 5, 6);
        fail("should fail with an exception");
    } catch (IndexOutOfBoundsException e) {
    // expected
    }
    // check length integer overflow
    in = handle.openInputStream();
    try {
        in.read(new byte[10], 5, Integer.MAX_VALUE);
        fail("should fail with an exception");
    } catch (IndexOutOfBoundsException e) {
    // expected
    }
}

Example 55 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class KeyedStateCheckpointOutputStreamTest method testReadWriteMissingKeyGroups.

@Test
public void testReadWriteMissingKeyGroups() throws Exception {
    final KeyGroupRange keyRange = new KeyGroupRange(0, 2);
    KeyedStateCheckpointOutputStream stream = createStream(keyRange);
    DataOutputView dov = new DataOutputViewStreamWrapper(stream);
    stream.startNewKeyGroup(1);
    dov.writeInt(1);
    KeyGroupsStateHandle fullHandle = stream.closeAndGetHandle();
    int count = 0;
    try (FSDataInputStream in = fullHandle.openInputStream()) {
        DataInputView div = new DataInputViewStreamWrapper(in);
        for (int kg : fullHandle.getKeyGroupRange()) {
            long off = fullHandle.getOffsetForKeyGroup(kg);
            if (off >= 0) {
                in.seek(off);
                Assert.assertEquals(1, div.readInt());
                ++count;
            }
        }
    }
    Assert.assertEquals(1, count);
}