Examples with FSDataOutputStream org.apache.flink.core.fs.FSDataOutputStream used on opensource projects

Example 26 with FSDataOutputStream

use of org.apache.flink.core.fs.FSDataOutputStream in project flink by apache.

the class LocalFileSystemTest method testLocalFilesystem.

/**
 * This test checks the functionality of the {@link LocalFileSystem} class.
 */
@Test
public void testLocalFilesystem() throws Exception {
    final File tempdir = new File(temporaryFolder.getRoot(), UUID.randomUUID().toString());
    final File testfile1 = new File(tempdir, UUID.randomUUID().toString());
    final File testfile2 = new File(tempdir, UUID.randomUUID().toString());
    final Path pathtotestfile1 = new Path(testfile1.toURI().getPath());
    final Path pathtotestfile2 = new Path(testfile2.toURI().getPath());
    final LocalFileSystem lfs = new LocalFileSystem();
    final Path pathtotmpdir = new Path(tempdir.toURI().getPath());
    /*
         * check that lfs can see/create/delete/read directories
         */
    // check that dir is not existent yet
    assertFalse(lfs.exists(pathtotmpdir));
    assertTrue(tempdir.mkdirs());
    // check that local file system recognizes file..
    assertTrue(lfs.exists(pathtotmpdir));
    final FileStatus localstatus1 = lfs.getFileStatus(pathtotmpdir);
    // check that lfs recognizes directory..
    assertTrue(localstatus1.isDir());
    // get status for files in this (empty) directory..
    final FileStatus[] statusforfiles = lfs.listStatus(pathtotmpdir);
    // no files in there.. hence, must be zero
    assertTrue(statusforfiles.length == 0);
    // check that lfs can delete directory..
    lfs.delete(pathtotmpdir, true);
    // double check that directory is not existent anymore..
    assertFalse(lfs.exists(pathtotmpdir));
    assertFalse(tempdir.exists());
    // re-create directory..
    lfs.mkdirs(pathtotmpdir);
    // creation successful?
    assertTrue(tempdir.exists());
    /*
         * check that lfs can create/read/write from/to files properly and read meta information..
         */
    // create files.. one ""natively"", one using lfs
    final FSDataOutputStream lfsoutput1 = lfs.create(pathtotestfile1, WriteMode.NO_OVERWRITE);
    assertTrue(testfile2.createNewFile());
    // does lfs create files? does lfs recognize created files?
    assertTrue(testfile1.exists());
    assertTrue(lfs.exists(pathtotestfile2));
    // test that lfs can write to files properly
    final byte[] testbytes = { 1, 2, 3, 4, 5 };
    lfsoutput1.write(testbytes);
    lfsoutput1.close();
    assertEquals(testfile1.length(), 5L);
    byte[] testbytestest = new byte[5];
    try (FileInputStream fisfile1 = new FileInputStream(testfile1)) {
        assertEquals(testbytestest.length, fisfile1.read(testbytestest));
    }
    assertArrayEquals(testbytes, testbytestest);
    // does lfs see the correct file length?
    assertEquals(lfs.getFileStatus(pathtotestfile1).getLen(), testfile1.length());
    // as well, when we call the listStatus (that is intended for directories?)
    assertEquals(lfs.listStatus(pathtotestfile1)[0].getLen(), testfile1.length());
    // test that lfs can read files properly
    final FileOutputStream fosfile2 = new FileOutputStream(testfile2);
    fosfile2.write(testbytes);
    fosfile2.close();
    testbytestest = new byte[5];
    final FSDataInputStream lfsinput2 = lfs.open(pathtotestfile2);
    assertEquals(lfsinput2.read(testbytestest), 5);
    lfsinput2.close();
    assertTrue(Arrays.equals(testbytes, testbytestest));
    // does lfs see two files?
    assertEquals(lfs.listStatus(pathtotmpdir).length, 2);
    // do we get exactly one blocklocation per file? no matter what start and len we provide
    assertEquals(lfs.getFileBlockLocations(lfs.getFileStatus(pathtotestfile1), 0, 0).length, 1);
    /*
         * can lfs delete files / directories?
         */
    assertTrue(lfs.delete(pathtotestfile1, false));
    // and can lfs also delete directories recursively?
    assertTrue(lfs.delete(pathtotmpdir, true));
    assertTrue(!tempdir.exists());
}

Example 27 with FSDataOutputStream

use of org.apache.flink.core.fs.FSDataOutputStream in project flink by apache.

the class FileSystemBlobStore method put.

private boolean put(File fromFile, String toBlobPath) throws IOException {
    try (FSDataOutputStream os = fileSystem.create(new Path(toBlobPath), FileSystem.WriteMode.OVERWRITE)) {
        LOG.debug("Copying from {} to {}.", fromFile, toBlobPath);
        Files.copy(fromFile, os);
        os.sync();
    }
    return true;
}

Example 28 with FSDataOutputStream

use of org.apache.flink.core.fs.FSDataOutputStream in project flink by apache.

the class CheckpointStateOutputStreamTest method testEmptyState.

// ------------------------------------------------------------------------
// Tests
// ------------------------------------------------------------------------
/**
 * Validates that even empty streams create a file and a file state handle.
 */
@Test
public void testEmptyState() throws Exception {
    final FileSystem fs = FileSystem.getLocalFileSystem();
    final Path folder = baseFolder();
    final String fileName = "myFileName";
    final Path filePath = new Path(folder, fileName);
    final FileStateHandle handle;
    try (FSDataOutputStream stream = createTestStream(fs, folder, fileName)) {
        handle = closeAndGetResult(stream);
    }
    // must have created a handle
    assertNotNull(handle);
    assertEquals(filePath, handle.getFilePath());
    // the pointer path should exist as a directory
    assertTrue(fs.exists(handle.getFilePath()));
    assertFalse(fs.getFileStatus(filePath).isDir());
    // the contents should be empty
    try (FSDataInputStream in = handle.openInputStream()) {
        assertEquals(-1, in.read());
    }
}

Example 29 with FSDataOutputStream

use of org.apache.flink.core.fs.FSDataOutputStream in project flink by apache.

the class CheckpointStateOutputStreamTest method testCleanupWhenClosingStream.

/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {
    final FileSystem fs = FileSystem.getLocalFileSystem();
    final Path folder = new Path(tmp.newFolder().toURI());
    final String fileName = "nonCreativeTestFileName";
    final Path path = new Path(folder, fileName);
    // write some test data and close the stream
    try (FSDataOutputStream stream = createTestStream(fs, folder, fileName)) {
        Random rnd = new Random();
        for (int i = 0; i < rnd.nextInt(1000); i++) {
            stream.write(rnd.nextInt(100));
        }
    }
    assertFalse(fs.exists(path));
}

Example 30 with FSDataOutputStream

use of org.apache.flink.core.fs.FSDataOutputStream in project flink by apache.

the class FsCheckpointStateOutputStreamTest method testCleanupWhenClosingStream.

/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {
    final FileSystem fs = mock(FileSystem.class);
    final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);
    final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
    when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);
    CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(Path.fromLocalFile(tempDir.newFolder()), fs, 4, 0, relativePaths);
    // this should create the underlying file stream
    stream.write(new byte[] { 1, 2, 3, 4, 5 });
    verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));
    stream.close();
    verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}