Examples with LocalDataOutputStream org.apache.flink.core.fs.local.LocalDataOutputStream used on opensource projects

Example 1 with LocalDataOutputStream

use of org.apache.flink.core.fs.local.LocalDataOutputStream in project flink by apache.

the class TestFileSystem method create.

@Override
public FSDataOutputStream create(final Path filePath, final WriteMode overwrite) throws IOException {
    currentUnclosedOutputStream.compute(filePath, (k, v) -> v == null ? 1 : v + 1);
    LocalDataOutputStream stream = (LocalDataOutputStream) super.create(filePath, overwrite);
    return new TestOutputStream(stream, filePath);
}

Example 2 with LocalDataOutputStream

use of org.apache.flink.core.fs.local.LocalDataOutputStream in project flink by apache.

the class OrcBulkWriterFactoryTest method testNotOverrideInMemoryManager.

@Test
public void testNotOverrideInMemoryManager() throws IOException {
    TestMemoryManager memoryManager = new TestMemoryManager();
    OrcBulkWriterFactory<Record> factory = new TestOrcBulkWriterFactory<>(new RecordVectorizer("struct<_col0:string,_col1:int>"), memoryManager);
    factory.create(new LocalDataOutputStream(temporaryFolder.newFile()));
    factory.create(new LocalDataOutputStream(temporaryFolder.newFile()));
    List<Path> addedWriterPath = memoryManager.getAddedWriterPath();
    assertEquals(2, addedWriterPath.size());
    assertNotEquals(addedWriterPath.get(0), addedWriterPath.get(1));
}

Example 3 with LocalDataOutputStream

use of org.apache.flink.core.fs.local.LocalDataOutputStream in project flink by apache.

the class InitOutputPathTest method runTest.

private void runTest(final boolean useAwaits) throws Exception {
    final File tempFile = tempDir.newFile();
    final Path path1 = new Path(tempFile.getAbsolutePath(), "1");
    final Path path2 = new Path(tempFile.getAbsolutePath(), "2");
    final OneShotLatch deleteAwaitLatch1 = new OneShotLatch();
    final OneShotLatch deleteAwaitLatch2 = new OneShotLatch();
    final OneShotLatch mkdirsAwaitLatch1 = new OneShotLatch();
    final OneShotLatch mkdirsAwaitLatch2 = new OneShotLatch();
    final OneShotLatch deleteTriggerLatch1 = new OneShotLatch();
    final OneShotLatch deletetriggerLatch2 = new OneShotLatch();
    final OneShotLatch mkdirsTriggerLatch1 = new OneShotLatch();
    final OneShotLatch mkdirsTriggerLatch2 = new OneShotLatch();
    final OneShotLatch createAwaitLatch = new OneShotLatch();
    final OneShotLatch createTriggerLatch = new OneShotLatch();
    // this "new LocalDataOutputStream()" is in the end called by the async threads
    whenNew(LocalDataOutputStream.class).withAnyArguments().thenAnswer(new Answer<LocalDataOutputStream>() {

        @Override
        public LocalDataOutputStream answer(InvocationOnMock invocation) throws Throwable {
            createAwaitLatch.trigger();
            createTriggerLatch.await();
            final File file = (File) invocation.getArguments()[0];
            return new LocalDataOutputStream(file);
        }
    });
    final LocalFileSystem fs1 = new SyncedFileSystem(deleteAwaitLatch1, mkdirsAwaitLatch1, deleteTriggerLatch1, mkdirsTriggerLatch1);
    final LocalFileSystem fs2 = new SyncedFileSystem(deleteAwaitLatch2, mkdirsAwaitLatch2, deletetriggerLatch2, mkdirsTriggerLatch2);
    // start the concurrent file creators
    FileCreator thread1 = new FileCreator(fs1, path1);
    FileCreator thread2 = new FileCreator(fs2, path2);
    thread1.start();
    thread2.start();
    // wait until they both decide to delete the directory
    if (useAwaits) {
        deleteAwaitLatch1.await();
        deleteAwaitLatch2.await();
    } else {
        Thread.sleep(5);
    }
    // now send off #1 to delete the directory (it will pass the 'mkdirs' fast) and wait to
    // create the file
    mkdirsTriggerLatch1.trigger();
    deleteTriggerLatch1.trigger();
    if (useAwaits) {
        createAwaitLatch.await();
    } else {
        // this needs a bit more sleep time, because here mockito is working
        Thread.sleep(100);
    }
    // now send off #2 to delete the directory - it waits at 'mkdirs'
    deletetriggerLatch2.trigger();
    if (useAwaits) {
        mkdirsAwaitLatch2.await();
    } else {
        Thread.sleep(5);
    }
    // let #1 try to create the file and see if it succeeded
    createTriggerLatch.trigger();
    if (useAwaits) {
        thread1.sync();
    } else {
        Thread.sleep(5);
    }
    // now let #1 finish up
    mkdirsTriggerLatch2.trigger();
    thread1.sync();
    thread2.sync();
}