Examples with FileOutStream alluxio.client.file.FileOutStream used on opensource projects

Example 21 with FileOutStream

use of alluxio.client.file.FileOutStream in project alluxio by Alluxio.

the class PersistMultipleMountsIntegrationTest method syncMultipleMountsDefaultPersist.

@Test
public void syncMultipleMountsDefaultPersist() throws Exception {
    // Skip non-local and non-HDFS UFSs.
    Assume.assumeTrue(UnderFileSystemUtils.isLocal(mUfs) || UnderFileSystemUtils.isHdfs(mUfs));
    String path = PathUtils.uniqPath();
    AlluxioURI filePath = new AlluxioURI(path);
    FileOutStream os = mFileSystem.createFile(filePath, CreateFileOptions.defaults().setWriteType(WriteType.CACHE_THROUGH));
    os.write((byte) 0);
    os.write((byte) 1);
    os.close();
    // Check the file is persisted
    URIStatus status = mFileSystem.getStatus(filePath);
    Assert.assertEquals(PersistenceState.PERSISTED.toString(), status.getPersistenceState());
    Assert.assertTrue(status.isCompleted());
    Assert.assertTrue(mUfs.exists(PathUtils.concatPath(mUfsRoot, path)));
    Assert.assertFalse(mMountedUfs.exists(PathUtils.concatPath(mMountedUfsRoot, path)));
}

Example 22 with FileOutStream

use of alluxio.client.file.FileOutStream in project alluxio by Alluxio.

the class PersistMultipleMountsIntegrationTest method syncMultipleMountsMountedPersist.

@Test
public void syncMultipleMountsMountedPersist() throws Exception {
    // Skip non-local and non-HDFS UFSs.
    Assume.assumeTrue(UnderFileSystemUtils.isLocal(mUfs) || UnderFileSystemUtils.isHdfs(mUfs));
    String path = PathUtils.uniqPath();
    AlluxioURI filePath = new AlluxioURI(MOUNT_PATH + path);
    FileOutStream os = mFileSystem.createFile(filePath, CreateFileOptions.defaults().setWriteType(WriteType.CACHE_THROUGH));
    os.write((byte) 0);
    os.write((byte) 1);
    os.close();
    // Check the file is persisted
    URIStatus status = mFileSystem.getStatus(filePath);
    Assert.assertEquals(PersistenceState.PERSISTED.toString(), status.getPersistenceState());
    Assert.assertTrue(status.isCompleted());
    Assert.assertFalse(mUfs.exists(PathUtils.concatPath(mUfsRoot, path)));
    Assert.assertTrue(mMountedUfs.exists(PathUtils.concatPath(mMountedUfsRoot, path)));
}

Example 23 with FileOutStream

use of alluxio.client.file.FileOutStream in project alluxio by Alluxio.

the class CpCommand method copyPath.

/**
   * Copies a file or directory specified by srcPath from the local filesystem to dstPath in the
   * Alluxio filesystem space.
   *
   * @param srcPath the {@link AlluxioURI} of the source file in the local filesystem
   * @param dstPath the {@link AlluxioURI} of the destination
   * @throws AlluxioException when Alluxio exception occurs
   * @throws IOException when non-Alluxio exception occurs
   */
private void copyPath(AlluxioURI srcPath, AlluxioURI dstPath) throws AlluxioException, IOException {
    File src = new File(srcPath.getPath());
    if (!src.isDirectory()) {
        // src will be copied to.
        if (mFileSystem.exists(dstPath) && mFileSystem.getStatus(dstPath).isFolder()) {
            dstPath = dstPath.join(src.getName());
        }
        FileOutStream os = null;
        try (Closer closer = Closer.create()) {
            os = closer.register(mFileSystem.createFile(dstPath));
            FileInputStream in = closer.register(new FileInputStream(src));
            FileChannel channel = closer.register(in.getChannel());
            ByteBuffer buf = ByteBuffer.allocate(8 * Constants.MB);
            while (channel.read(buf) != -1) {
                buf.flip();
                os.write(buf.array(), 0, buf.limit());
            }
        } catch (Exception e) {
            // around.
            if (os != null) {
                os.cancel();
                if (mFileSystem.exists(dstPath)) {
                    mFileSystem.delete(dstPath);
                }
            }
            throw e;
        }
    } else {
        mFileSystem.createDirectory(dstPath);
        List<String> errorMessages = new ArrayList<>();
        File[] fileList = src.listFiles();
        if (fileList == null) {
            String errMsg = String.format("Failed to list files for directory %s", src);
            errorMessages.add(errMsg);
            fileList = new File[0];
        }
        int misFiles = 0;
        for (File srcFile : fileList) {
            AlluxioURI newURI = new AlluxioURI(dstPath, new AlluxioURI(srcFile.getName()));
            try {
                copyPath(new AlluxioURI(srcPath.getScheme(), srcPath.getAuthority(), srcFile.getPath()), newURI);
            } catch (IOException e) {
                errorMessages.add(e.getMessage());
                if (!mFileSystem.exists(newURI)) {
                    misFiles++;
                }
            }
        }
        if (errorMessages.size() != 0) {
            if (misFiles == fileList.length) {
                // If the directory doesn't exist and no files were created, then delete the directory
                if (mFileSystem.exists(dstPath)) {
                    mFileSystem.delete(dstPath);
                }
            }
            throw new IOException(Joiner.on('\n').join(errorMessages));
        }
    }
}

Example 24 with FileOutStream

use of alluxio.client.file.FileOutStream in project alluxio by Alluxio.

the class RemoteBlockInStreamIntegrationTest method readMultiBlockFile.

/**
   * Tests that reading a file consisting of more than one block from the underfs works.
   */
@Test
public void readMultiBlockFile() throws Exception {
    String uniqPath = PathUtils.uniqPath();
    int blockSizeByte = 10;
    int numBlocks = 10;
    AlluxioURI uri = new AlluxioURI(uniqPath);
    FileOutStream os = mFileSystem.createFile(uri, mWriteUnderStore);
    for (int i = 0; i < numBlocks; i++) {
        for (int j = 0; j < blockSizeByte; j++) {
            os.write((byte) (i * blockSizeByte + j));
        }
    }
    os.close();
    FileInStream is = mFileSystem.openFile(uri, mReadCache);
    for (int i = 0; i < blockSizeByte * numBlocks; i++) {
        Assert.assertEquals((byte) i, is.read());
    }
    is.close();
    Assert.assertTrue(mFileSystem.getStatus(uri).getInMemoryPercentage() == 100);
}

Example 25 with FileOutStream

use of alluxio.client.file.FileOutStream in project alluxio by Alluxio.

the class UnderFileSystemBlockInStreamIntegrationTest method readMultiBlockFile.

/**
   * Tests that reading a file consisting of more than one block from the underfs works.
   */
@Test
public void readMultiBlockFile() throws Exception {
    String uniqPath = PathUtils.uniqPath();
    int blockSizeByte = 10;
    int numBlocks = 10;
    AlluxioURI uri = new AlluxioURI(uniqPath);
    FileOutStream os = mFileSystem.createFile(uri, mWriteUnderStore);
    for (int i = 0; i < numBlocks; i++) {
        for (int j = 0; j < blockSizeByte; j++) {
            os.write((byte) (i * blockSizeByte + j));
        }
    }
    os.close();
    FileInStream is = mFileSystem.openFile(uri, mReadCache);
    for (int i = 0; i < blockSizeByte * numBlocks; i++) {
        Assert.assertEquals((byte) i, is.read());
    }
    is.close();
    Assert.assertTrue(mFileSystem.getStatus(uri).getInMemoryPercentage() == 100);
}