Examples with FSDataInputStream org.apache.flink.core.fs.FSDataInputStream used on opensource projects

Example 41 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class FileUtils method internalCopyFile.

private static void internalCopyFile(Path sourcePath, Path targetPath, boolean executable, FileSystem sFS, FileSystem tFS) throws IOException {
    try (FSDataOutputStream lfsOutput = tFS.create(targetPath, FileSystem.WriteMode.NO_OVERWRITE);
        FSDataInputStream fsInput = sFS.open(sourcePath)) {
        IOUtils.copyBytes(fsInput, lfsOutput);
        // noinspection ResultOfMethodCallIgnored
        new File(targetPath.toString()).setExecutable(executable);
    }
}

Example 42 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class LocalFileSystemTest method testLocalFilesystem.

/**
 * This test checks the functionality of the {@link LocalFileSystem} class.
 */
@Test
public void testLocalFilesystem() throws Exception {
    final File tempdir = new File(temporaryFolder.getRoot(), UUID.randomUUID().toString());
    final File testfile1 = new File(tempdir, UUID.randomUUID().toString());
    final File testfile2 = new File(tempdir, UUID.randomUUID().toString());
    final Path pathtotestfile1 = new Path(testfile1.toURI().getPath());
    final Path pathtotestfile2 = new Path(testfile2.toURI().getPath());
    final LocalFileSystem lfs = new LocalFileSystem();
    final Path pathtotmpdir = new Path(tempdir.toURI().getPath());
    /*
         * check that lfs can see/create/delete/read directories
         */
    // check that dir is not existent yet
    assertFalse(lfs.exists(pathtotmpdir));
    assertTrue(tempdir.mkdirs());
    // check that local file system recognizes file..
    assertTrue(lfs.exists(pathtotmpdir));
    final FileStatus localstatus1 = lfs.getFileStatus(pathtotmpdir);
    // check that lfs recognizes directory..
    assertTrue(localstatus1.isDir());
    // get status for files in this (empty) directory..
    final FileStatus[] statusforfiles = lfs.listStatus(pathtotmpdir);
    // no files in there.. hence, must be zero
    assertTrue(statusforfiles.length == 0);
    // check that lfs can delete directory..
    lfs.delete(pathtotmpdir, true);
    // double check that directory is not existent anymore..
    assertFalse(lfs.exists(pathtotmpdir));
    assertFalse(tempdir.exists());
    // re-create directory..
    lfs.mkdirs(pathtotmpdir);
    // creation successful?
    assertTrue(tempdir.exists());
    /*
         * check that lfs can create/read/write from/to files properly and read meta information..
         */
    // create files.. one ""natively"", one using lfs
    final FSDataOutputStream lfsoutput1 = lfs.create(pathtotestfile1, WriteMode.NO_OVERWRITE);
    assertTrue(testfile2.createNewFile());
    // does lfs create files? does lfs recognize created files?
    assertTrue(testfile1.exists());
    assertTrue(lfs.exists(pathtotestfile2));
    // test that lfs can write to files properly
    final byte[] testbytes = { 1, 2, 3, 4, 5 };
    lfsoutput1.write(testbytes);
    lfsoutput1.close();
    assertEquals(testfile1.length(), 5L);
    byte[] testbytestest = new byte[5];
    try (FileInputStream fisfile1 = new FileInputStream(testfile1)) {
        assertEquals(testbytestest.length, fisfile1.read(testbytestest));
    }
    assertArrayEquals(testbytes, testbytestest);
    // does lfs see the correct file length?
    assertEquals(lfs.getFileStatus(pathtotestfile1).getLen(), testfile1.length());
    // as well, when we call the listStatus (that is intended for directories?)
    assertEquals(lfs.listStatus(pathtotestfile1)[0].getLen(), testfile1.length());
    // test that lfs can read files properly
    final FileOutputStream fosfile2 = new FileOutputStream(testfile2);
    fosfile2.write(testbytes);
    fosfile2.close();
    testbytestest = new byte[5];
    final FSDataInputStream lfsinput2 = lfs.open(pathtotestfile2);
    assertEquals(lfsinput2.read(testbytestest), 5);
    lfsinput2.close();
    assertTrue(Arrays.equals(testbytes, testbytestest));
    // does lfs see two files?
    assertEquals(lfs.listStatus(pathtotmpdir).length, 2);
    // do we get exactly one blocklocation per file? no matter what start and len we provide
    assertEquals(lfs.getFileBlockLocations(lfs.getFileStatus(pathtotestfile1), 0, 0).length, 1);
    /*
         * can lfs delete files / directories?
         */
    assertTrue(lfs.delete(pathtotestfile1, false));
    // and can lfs also delete directories recursively?
    assertTrue(lfs.delete(pathtotmpdir, true));
    assertTrue(!tempdir.exists());
}

Example 43 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class AvroParquetRecordFormatTest method createReader.

// ------------------------------------------------------------------------
// helper methods
// ------------------------------------------------------------------------
private <T> StreamFormat.Reader<T> createReader(AvroParquetRecordFormat<T> format, Configuration config, Path filePath, long splitOffset, long splitLength) throws IOException {
    final FileSystem fileSystem = filePath.getFileSystem();
    final FileStatus fileStatus = fileSystem.getFileStatus(filePath);
    final FSDataInputStream inputStream = fileSystem.open(filePath);
    if (format.isSplittable()) {
        inputStream.seek(splitOffset);
    } else {
        inputStream.seek(0);
        checkArgument(splitLength == fileStatus.getLen());
    }
    return format.createReader(config, inputStream, fileStatus.getLen(), splitOffset + splitLength);
}

Example 44 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class FileReadFunction method flatMap.

@Override
public void flatMap(Tuple3<String, Long, Long> value, Collector<String> out) throws Exception {
    FSDataInputStream stream = FileSystem.get(new URI(value.f0)).open(new Path(value.f0));
    stream.seek(value.f1);
    BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
    String line;
    try {
        while ((line = reader.readLine()) != null && (value.f2 == -1L || stream.getPos() <= value.f2)) {
            out.collect(line);
        }
    } finally {
        reader.close();
    }
}

Example 45 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class HeapRestoreOperation method readStateHandleStateData.

private void readStateHandleStateData(FSDataInputStream fsDataInputStream, DataInputViewStreamWrapper inView, KeyGroupRangeOffsets keyGroupOffsets, Map<Integer, StateMetaInfoSnapshot> kvStatesById, int numStates, int readVersion, boolean isCompressed) throws IOException {
    final StreamCompressionDecorator streamCompressionDecorator = isCompressed ? SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;
    for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) {
        int keyGroupIndex = groupOffset.f0;
        long offset = groupOffset.f1;
        if (!keyGroupRange.contains(keyGroupIndex)) {
            LOG.debug("Key group {} doesn't belong to this backend with key group range: {}", keyGroupIndex, keyGroupRange);
            continue;
        }
        fsDataInputStream.seek(offset);
        int writtenKeyGroupIndex = inView.readInt();
        Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex, "Unexpected key-group in restore.");
        try (InputStream kgCompressionInStream = streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) {
            readKeyGroupStateData(kgCompressionInStream, kvStatesById, keyGroupIndex, numStates, readVersion);
        }
    }
}