Examples with FSDataInputStream org.apache.flink.core.fs.FSDataInputStream used on opensource projects

Example 56 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class KeyedStateCheckpointOutputStreamTest method verifyRead.

private static void verifyRead(KeyGroupsStateHandle fullHandle, KeyGroupRange keyRange) throws IOException {
    int count = 0;
    try (FSDataInputStream in = fullHandle.openInputStream()) {
        DataInputView div = new DataInputViewStreamWrapper(in);
        for (int kg : fullHandle.getKeyGroupRange()) {
            long off = fullHandle.getOffsetForKeyGroup(kg);
            in.seek(off);
            Assert.assertEquals(kg, div.readInt());
            ++count;
        }
    }
    Assert.assertEquals(keyRange.getNumberOfKeyGroups(), count);
}

Example 57 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class OperatorStateOutputCheckpointStreamTest method verifyRead.

private static void verifyRead(OperatorStateHandle fullHandle, int numPartitions) throws IOException {
    int count = 0;
    try (FSDataInputStream in = fullHandle.openInputStream()) {
        OperatorStateHandle.StateMetaInfo metaInfo = fullHandle.getStateNameToPartitionOffsets().get(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME);
        long[] offsets = metaInfo.getOffsets();
        Assert.assertNotNull(offsets);
        DataInputView div = new DataInputViewStreamWrapper(in);
        for (int i = 0; i < numPartitions; ++i) {
            in.seek(offsets[i]);
            Assert.assertEquals(i, div.readInt());
            ++count;
        }
    }
    Assert.assertEquals(numPartitions, count);
}

Example 58 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class StreamFormatAdapter method openStream.

private static TrackingFsDataInputStream openStream(final Path file, final Configuration config, final long seekPosition) throws IOException {
    final FileSystem fs = file.getFileSystem();
    final long fileLength = fs.getFileStatus(file).getLen();
    final int fetchSize = MathUtils.checkedDownCast(config.get(StreamFormat.FETCH_IO_SIZE).getBytes());
    if (fetchSize <= 0) {
        throw new IllegalConfigurationException(String.format("The fetch size (%s) must be > 0, but is %d", StreamFormat.FETCH_IO_SIZE.key(), fetchSize));
    }
    final InflaterInputStreamFactory<?> deCompressor = StandardDeCompressors.getDecompressorForFileName(file.getPath());
    final FSDataInputStream inStream = fs.open(file);
    return doWithCleanupOnException(inStream, () -> {
        final FSDataInputStream in = deCompressor == null ? inStream : new InputStreamFSInputWrapper(deCompressor.create(inStream));
        in.seek(seekPosition);
        return new TrackingFsDataInputStream(in, fileLength, fetchSize);
    });
}