Examples with FileOutStream alluxio.client.file.FileOutStream used on opensource projects

Example 6 with FileOutStream

use of alluxio.client.file.FileOutStream in project alluxio by Alluxio.

the class BasicNonByteBufferOperations method write.

private void write(FileSystem alluxioClient) throws IOException, AlluxioException {
    FileOutStream fileOutStream = createFile(alluxioClient, mFilePath, mDeleteIfExists);
    long startTimeMs = CommonUtils.getCurrentMs();
    try (DataOutputStream os = new DataOutputStream(fileOutStream)) {
        os.writeInt(mLength);
        for (int i = 0; i < mLength; i++) {
            os.writeInt(i);
        }
    }
    LOG.info(FormatUtils.formatTimeTakenMs(startTimeMs, "writeFile to file " + mFilePath));
}

Example 7 with FileOutStream

use of alluxio.client.file.FileOutStream in project alluxio by Alluxio.

the class BasicOperations method writeFile.

private void writeFile(FileSystem fileSystem) throws IOException, AlluxioException {
    ByteBuffer buf = ByteBuffer.allocate(NUMBERS * 4);
    buf.order(ByteOrder.nativeOrder());
    for (int k = 0; k < NUMBERS; k++) {
        buf.putInt(k);
    }
    LOG.debug("Writing data...");
    long startTimeMs = CommonUtils.getCurrentMs();
    FileOutStream os = fileSystem.createFile(mFilePath, mWriteOptions);
    os.write(buf.array());
    os.close();
    LOG.info(FormatUtils.formatTimeTakenMs(startTimeMs, "writeFile to file " + mFilePath));
}

Example 8 with FileOutStream

use of alluxio.client.file.FileOutStream in project alluxio by Alluxio.

the class MultiMount method main.

/**
   * Entry point for the {@link MultiMount} program.
   *
   * @param args command-line arguments
   */
public static void main(String[] args) {
    if (args.length != 1) {
        System.err.println("Usage: ./bin/alluxio runClass alluxio.examples.MultiMount <HDFS_URL>");
        System.exit(-1);
    }
    AlluxioURI mntPath = new AlluxioURI("/mnt");
    AlluxioURI s3Mount = new AlluxioURI("/mnt/s3");
    AlluxioURI inputPath = new AlluxioURI("/mnt/s3/hello.txt");
    AlluxioURI s3Path = new AlluxioURI("s3n://alluxio-demo/");
    AlluxioURI hdfsMount = new AlluxioURI("/mnt/hdfs");
    AlluxioURI outputPath = new AlluxioURI("/mnt/hdfs/hello.txt");
    AlluxioURI hdfsPath = new AlluxioURI(args[0]);
    FileSystem fileSystem = FileSystem.Factory.get();
    try {
        // Make sure mount directory exists.
        if (!fileSystem.exists(mntPath)) {
            System.out.print("creating " + mntPath + " ... ");
            fileSystem.createDirectory(mntPath);
            System.out.println("done");
        }
        // Make sure the S3 mount point does not exist.
        if (fileSystem.exists(s3Mount)) {
            System.out.print("unmounting " + s3Mount + " ... ");
            fileSystem.unmount(s3Mount);
            System.out.println("done");
        }
        // Make sure the HDFS mount point does not exist.
        if (fileSystem.exists(hdfsMount)) {
            System.out.print("unmounting " + hdfsMount + " ... ");
            fileSystem.unmount(hdfsMount);
            System.out.println("done");
        }
        // Mount S3.
        System.out.print("mounting " + s3Path + " to " + s3Mount + " ... ");
        fileSystem.mount(s3Mount, s3Path);
        System.out.println("done");
        // Mount HDFS.
        System.out.print("mounting " + hdfsPath + " to " + hdfsMount + " ... ");
        fileSystem.mount(hdfsMount, hdfsPath);
        System.out.println("done");
        // Make sure output file does not exist.
        if (fileSystem.exists(outputPath)) {
            System.out.print("deleting " + outputPath + " ... ");
            fileSystem.delete(outputPath);
            System.out.println("done");
        }
        // Open the input stream.
        System.out.print("opening " + inputPath + " ... ");
        FileInStream is = fileSystem.openFile(inputPath);
        System.out.println("done");
        // Open the output stream, setting the write type to make sure result is persisted.
        System.out.print("opening " + outputPath + " ... ");
        CreateFileOptions options = CreateFileOptions.defaults().setWriteType(WriteType.CACHE_THROUGH);
        FileOutStream os = fileSystem.createFile(outputPath, options);
        System.out.println("done");
        // Copy the data
        System.out.print("transferring data from " + inputPath + " to " + outputPath + " ... ");
        IOUtils.copy(is, os);
        System.out.println("done");
        // Close the input stream.
        System.out.print("closing " + inputPath + " ... ");
        is.close();
        System.out.println("done");
        // Close the output stream.
        System.out.print("closing " + outputPath + " ... ");
        os.close();
        System.out.println("done");
    } catch (Exception e) {
        System.out.println("fail");
        e.printStackTrace();
    } finally {
        // Make sure the S3 mount point is removed.
        try {
            if (fileSystem.exists(s3Mount)) {
                System.out.print("unmounting " + s3Mount + " ... ");
                fileSystem.unmount(s3Mount);
                System.out.println("done");
            }
        } catch (Exception e) {
            System.out.println("fail");
            e.printStackTrace();
        }
        // Make sure the HDFS mount point is removed.
        try {
            if (fileSystem.exists(hdfsMount)) {
                System.out.print("unmounting " + hdfsMount + " ... ");
                fileSystem.unmount(hdfsMount);
                System.out.println("done");
            }
        } catch (Exception e) {
            System.out.println("fail");
            e.printStackTrace();
        }
    }
}

Example 9 with FileOutStream

use of alluxio.client.file.FileOutStream in project alluxio by Alluxio.

the class FileSystemUtilsIntegrationTest method waitCompletedTest2.

@Test
public void waitCompletedTest2() throws IOException, AlluxioException, InterruptedException {
    final String uniqPath = PathUtils.uniqPath();
    // random value chosen through a fair dice roll :P
    final int numWrites = 4;
    final AlluxioURI uri = new AlluxioURI(uniqPath);
    final Runnable writer = new Runnable() {

        @Override
        public void run() {
            try {
                FileOutStream os = sFileSystem.createFile(uri, sWriteBoth);
                boolean completed = sFileSystem.getStatus(uri).isCompleted();
                Assert.assertFalse(completed);
                // four writes that will take > 600ms due to the sleeps
                for (int i = 0; i < numWrites; i++) {
                    os.write(42);
                    CommonUtils.sleepMs(200);
                }
                os.close();
                completed = sFileSystem.getStatus(uri).isCompleted();
                Assert.assertTrue(completed);
            } catch (Exception e) {
                Assert.fail(e.getMessage());
            }
        }
    };
    final Runnable waiter = new Runnable() {

        @Override
        public void run() {
            try {
                // set the slow default polling period to a more sensible value, in order
                // to speed up the tests artificial waiting times
                String original = Configuration.get(PropertyKey.USER_FILE_WAITCOMPLETED_POLL_MS);
                Configuration.set(PropertyKey.USER_FILE_WAITCOMPLETED_POLL_MS, "100");
                try {
                    // The write will take at most 600ms I am waiting for at most 400ms - epsilon.
                    boolean completed = FileSystemUtils.waitCompleted(sFileSystem, uri, 300, TimeUnit.MILLISECONDS);
                    Assert.assertFalse(completed);
                    completed = sFileSystem.getStatus(uri).isCompleted();
                    Assert.assertFalse(completed);
                } finally {
                    Configuration.set(PropertyKey.USER_FILE_WAITCOMPLETED_POLL_MS, original);
                }
            } catch (Exception e) {
                e.printStackTrace();
                Assert.fail(e.getMessage());
            }
        }
    };
    final Thread waitingThread = new Thread(waiter);
    waitingThread.start();
    final Thread writingThread = new Thread(writer);
    writingThread.start();
    waitingThread.join();
    writingThread.join();
}

Example 10 with FileOutStream

use of alluxio.client.file.FileOutStream in project alluxio by Alluxio.

the class FileSystemUtilsIntegrationTest method waitCompletedTest1.

@Test
public void waitCompletedTest1() throws IOException, AlluxioException, InterruptedException {
    final String uniqPath = PathUtils.uniqPath();
    // random value chosen through a fair dice roll :P
    final int numWrites = 4;
    final AlluxioURI uri = new AlluxioURI(uniqPath);
    final Runnable writer = new Runnable() {

        @Override
        public void run() {
            try {
                FileOutStream os = sFileSystem.createFile(uri, sWriteBoth);
                boolean completed = sFileSystem.getStatus(uri).isCompleted();
                Assert.assertFalse(completed);
                for (int i = 0; i < numWrites; i++) {
                    os.write(42);
                    CommonUtils.sleepMs(200);
                }
                os.close();
                completed = sFileSystem.getStatus(uri).isCompleted();
                Assert.assertTrue(completed);
            } catch (Exception e) {
                Assert.fail(e.getMessage());
            }
        }
    };
    final Runnable waiter = new Runnable() {

        @Override
        public void run() {
            try {
                boolean completed = FileSystemUtils.waitCompleted(sFileSystem, uri);
                Assert.assertTrue(completed);
                completed = sFileSystem.getStatus(uri).isCompleted();
                Assert.assertTrue(completed);
            } catch (Exception e) {
                e.printStackTrace();
                Assert.fail(e.getMessage());
            }
        }
    };
    final Thread waitingThread = new Thread(waiter);
    waitingThread.start();
    final Thread writingThread = new Thread(writer);
    writingThread.start();
    waitingThread.join();
    writingThread.join();
}