Examples with FileOutStream alluxio.client.file.FileOutStream used on opensource projects

Example 36 with FileOutStream

use of alluxio.client.file.FileOutStream in project alluxio by Alluxio.

the class LineageMasterIntegrationTest method lineageRecovery.

/**
   * Tests that a lineage job is executed when the output file for the lineage is reported as lost.
   *
   * The checkpoint interval is set high so that we are guaranteed to call reportLostFile
   * before persistence is complete.
   */
@Test(timeout = 100000)
@LocalAlluxioClusterResource.Config(confParams = { PropertyKey.Name.MASTER_LINEAGE_CHECKPOINT_INTERVAL_MS, "100000" })
public void lineageRecovery() throws Exception {
    final File logFile = mFolder.newFile();
    // Delete the log file so that when it starts to exist we know that it was created by the
    // lineage recompute job
    logFile.delete();
    FileSystem fs = FileSystem.Factory.get();
    try (LineageMasterClient lineageClient = getLineageMasterClient()) {
        lineageClient.createLineage(ImmutableList.<String>of(), ImmutableList.of("/testFile"), new CommandLineJob("echo hello world", new JobConf(logFile.getAbsolutePath())));
        FileOutStream out = fs.createFile(new AlluxioURI("/testFile"));
        out.write("foo".getBytes());
        out.close();
        lineageClient.reportLostFile("/testFile");
    }
    // Wait for the log file to be created by the recompute job
    CommonUtils.waitFor("the log file to be written", new Function<Void, Boolean>() {

        @Override
        public Boolean apply(Void input) {
            if (!logFile.exists()) {
                return false;
            }
            try (BufferedReader reader = new BufferedReader(new FileReader(logFile))) {
                String line = reader.readLine();
                return line != null && line.equals("hello world");
            } catch (Exception e) {
                throw Throwables.propagate(e);
            }
        }
    }, WaitForOptions.defaults().setTimeout(100 * Constants.SECOND_MS));
}

Example 37 with FileOutStream

use of alluxio.client.file.FileOutStream in project alluxio by Alluxio.

the class LineageMasterIntegrationTest method lineageCompleteAndAsyncPersist.

@Test
public void lineageCompleteAndAsyncPersist() throws Exception {
    try (LineageMasterClient lineageMasterClient = getLineageMasterClient()) {
        ArrayList<String> outFiles = new ArrayList<>();
        Collections.addAll(outFiles, OUT_FILE);
        lineageMasterClient.createLineage(new ArrayList<String>(), outFiles, mJob);
        CreateFileOptions options = CreateFileOptions.defaults().setWriteType(WriteType.MUST_CACHE).setBlockSizeBytes(BLOCK_SIZE_BYTES);
        LineageFileSystem fs = (LineageFileSystem) mLocalAlluxioClusterResource.get().getClient();
        FileOutStream outputStream = fs.createFile(new AlluxioURI(OUT_FILE), options);
        outputStream.write(1);
        outputStream.close();
        List<LineageInfo> infos = lineageMasterClient.getLineageInfoList();
        AlluxioURI uri = new AlluxioURI(infos.get(0).getOutputFiles().get(0));
        URIStatus status = getFileSystemMasterClient().getStatus(uri);
        Assert.assertNotEquals(PersistenceState.PERSISTED.toString(), status.getPersistenceState());
        Assert.assertTrue(status.isCompleted());
        IntegrationTestUtils.waitForPersist(mLocalAlluxioClusterResource, uri);
        // worker notifies the master
        status = getFileSystemMasterClient().getStatus(uri);
        Assert.assertEquals(PersistenceState.PERSISTED.toString(), status.getPersistenceState());
    }
}

Example 38 with FileOutStream

use of alluxio.client.file.FileOutStream in project alluxio by Alluxio.

the class LineageFileSystemTest method getNonLineageStream.

/**
   * Tests that a {@link FileOutStream} is returned.
   */
@Test
public void getNonLineageStream() throws Exception {
    AlluxioURI path = new AlluxioURI("test");
    Mockito.when(mLineageMasterClient.reinitializeFile("test", TEST_BLOCK_SIZE, 0, TtlAction.DELETE)).thenThrow(new LineageDoesNotExistException("lineage does not exist"));
    CreateFileOptions options = CreateFileOptions.defaults().setBlockSizeBytes(TEST_BLOCK_SIZE).setTtl(0);
    FileOutStream outStream = mAlluxioLineageFileSystem.createFile(path, options);
    Assert.assertTrue(outStream != null);
    Assert.assertFalse(outStream instanceof LineageFileOutStream);
    Assert.assertFalse(outStream instanceof DummyFileOutputStream);
    // verify client is released
    Mockito.verify(mLineageContext).releaseMasterClient(mLineageMasterClient);
}

Example 39 with FileOutStream

use of alluxio.client.file.FileOutStream in project alluxio by Alluxio.

the class AlluxioFrameworkIntegrationTest method basicAlluxioTests.

private static void basicAlluxioTests() throws Exception {
    LOG.info("Running tests");
    FileSystem fs = FileSystem.Factory.get();
    int listSize = fs.listStatus(new AlluxioURI("/")).size();
    if (listSize != 1) {
        throw new RuntimeException("Expected 1 path to exist at the root, but found " + listSize);
    }
    FileOutStream outStream = fs.createFile(new AlluxioURI("/test"));
    outStream.write("abc".getBytes());
    outStream.close();
    FileInStream inStream = fs.openFile(new AlluxioURI("/test"));
    String result = IOUtils.toString(inStream);
    if (!result.equals("abc")) {
        throw new RuntimeException("Expected abc but got " + result);
    }
    LOG.info("Tests passed");
}