Examples with FileSystem alluxio.client.file.FileSystem used on opensource projects

Example 41 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class MiniBenchmark method writeFile.

/**
 * Writes a file.
 *
 * @param count the count to determine the filename
 * @throws Exception if it fails to write
 */
private static void writeFile(CyclicBarrier barrier, AtomicLong runtime, int count, AlluxioConfiguration alluxioConf) throws Exception {
    FileSystem fileSystem = FileSystem.Factory.create(alluxioConf);
    byte[] buffer = new byte[(int) Math.min(sFileSize, 4 * Constants.MB)];
    Arrays.fill(buffer, (byte) 'a');
    AlluxioURI path = filename(count);
    if (fileSystem.exists(path)) {
        fileSystem.delete(path);
    }
    barrier.await();
    long startTime = System.nanoTime();
    long bytesWritten = 0;
    try (FileOutStream outStream = fileSystem.createFile(path)) {
        while (bytesWritten < sFileSize) {
            outStream.write(buffer, 0, (int) Math.min(buffer.length, sFileSize - bytesWritten));
            bytesWritten += buffer.length;
        }
    }
    runtime.addAndGet(System.nanoTime() - startTime);
}

Example 42 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class ManagerProcessContextTest method testApplyExistingMount.

@Test
public void testApplyExistingMount() throws Exception {
    FileSystem mockFs = mock(FileSystem.class);
    Map<String, MountPointInfo> mpi = new HashMap<>();
    mpi.put("/", new MountPointInfo().setUfsUri("hdfs://localhost:9000/").setReadOnly(true).setUfsType("hdfs").setUfsCapacityBytes(9001));
    doReturn(mockFs).when(mContext).getFileSystem();
    doReturn(mpi).when(mockFs).getMountTable();
    ApplyMountPointRequest vr = ApplyMountPointRequest.newBuilder().setPayload(ApplyMountPointRequest.Payload.newBuilder().setNew(mContext.getMountPointList().getMountPointList().get(0)).setBefore(mContext.getMountPointList().getMountPointList().get(0))).build();
    mContext.applyMount(vr);
}

Example 43 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class ManagerProcessContextTest method testApplyRootMountToOtherPath.

@Test
public void testApplyRootMountToOtherPath() throws Exception {
    FileSystem mockFs = mock(FileSystem.class);
    Map<String, MountPointInfo> mpi = new HashMap<>();
    mpi.put("/", new MountPointInfo().setUfsUri("hdfs://localhost:9000/").setReadOnly(true).setUfsType("hdfs").setUfsCapacityBytes(9001));
    doReturn(mockFs).when(mContext).getFileSystem();
    doReturn(mpi).when(mockFs).getMountTable();
    ApplyMountPointRequest vr = ApplyMountPointRequest.newBuilder().setPayload(ApplyMountPointRequest.Payload.newBuilder().setNew(mContext.getMountPointList().getMountPointList().get(0).toBuilder().setAlluxioPath("/mnt/new")).setBefore(mContext.getMountPointList().getMountPointList().get(0))).build();
    ApplyMountPointResponse.Payload resp = mContext.applyMount(vr);
    assertFalse(resp.getSuccess());
    assertTrue(resp.getErrorCount() > 0);
}

Example 44 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class ManagerProcessContextTest method testListMountPoints.

@Test
public void testListMountPoints() throws Exception {
    FileSystem mockFs = mock(FileSystem.class);
    Map<String, MountPointInfo> mpi = new HashMap<>();
    mpi.put("/", new MountPointInfo().setUfsUri("hdfs://localhost:9000/").setReadOnly(true).setUfsType("hdfs").setUfsCapacityBytes(9001));
    doReturn(mockFs).when(mContext).getFileSystem();
    doReturn(mpi).when(mockFs).getMountTable();
    ListMountPointResponse.Payload mpl = mContext.getMountPointList();
    assertEquals(1, mpl.getMountPointList().size());
    alluxio.hub.proto.MountPointInfo i = mpl.getMountPointList().get(0);
    assertEquals("hdfs://localhost:9000/", i.getUfsUri());
    assertTrue(i.getReadOnly());
    assertEquals("/", i.getAlluxioPath());
    assertEquals(0, i.getPropertiesCount());
}

Example 45 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class MultiMount method main.

/**
 * Entry point for the {@link MultiMount} program.
 *
 * @param args command-line arguments
 */
public static void main(String[] args) {
    if (args.length != 1) {
        System.err.println("Usage: ./bin/alluxio runClass alluxio.examples.MultiMount <HDFS_URL>");
        System.exit(-1);
    }
    AlluxioConfiguration alluxioConf = new InstancedConfiguration(ConfigurationUtils.defaults());
    AlluxioURI mntPath = new AlluxioURI("/mnt");
    AlluxioURI s3Mount = new AlluxioURI("/mnt/s3");
    AlluxioURI inputPath = new AlluxioURI("/mnt/s3/hello.txt");
    AlluxioURI s3Path = new AlluxioURI("s3a://alluxio-demo/");
    AlluxioURI hdfsMount = new AlluxioURI("/mnt/hdfs");
    AlluxioURI outputPath = new AlluxioURI("/mnt/hdfs/hello.txt");
    AlluxioURI hdfsPath = new AlluxioURI(args[0]);
    FileSystem fileSystem = FileSystem.Factory.create(alluxioConf);
    try {
        // Make sure mount directory exists.
        if (!fileSystem.exists(mntPath)) {
            System.out.print("creating " + mntPath + " ... ");
            fileSystem.createDirectory(mntPath);
            System.out.println("done");
        }
        // Make sure the S3 mount point does not exist.
        if (fileSystem.exists(s3Mount)) {
            System.out.print("unmounting " + s3Mount + " ... ");
            fileSystem.unmount(s3Mount);
            System.out.println("done");
        }
        // Make sure the HDFS mount point does not exist.
        if (fileSystem.exists(hdfsMount)) {
            System.out.print("unmounting " + hdfsMount + " ... ");
            fileSystem.unmount(hdfsMount);
            System.out.println("done");
        }
        // Mount S3.
        System.out.print("mounting " + s3Path + " to " + s3Mount + " ... ");
        fileSystem.mount(s3Mount, s3Path);
        System.out.println("done");
        // Mount HDFS.
        System.out.print("mounting " + hdfsPath + " to " + hdfsMount + " ... ");
        fileSystem.mount(hdfsMount, hdfsPath);
        System.out.println("done");
        // Make sure output file does not exist.
        if (fileSystem.exists(outputPath)) {
            System.out.print("deleting " + outputPath + " ... ");
            fileSystem.delete(outputPath);
            System.out.println("done");
        }
        // Open the input stream.
        System.out.print("opening " + inputPath + " ... ");
        FileInStream is = fileSystem.openFile(inputPath);
        System.out.println("done");
        // Open the output stream, setting the write type to make sure result is persisted.
        System.out.print("opening " + outputPath + " ... ");
        CreateFilePOptions options = CreateFilePOptions.newBuilder().setWriteType(WritePType.CACHE_THROUGH).setRecursive(true).build();
        FileOutStream os = fileSystem.createFile(outputPath, options);
        System.out.println("done");
        // Copy the data
        System.out.print("transferring data from " + inputPath + " to " + outputPath + " ... ");
        IOUtils.copy(is, os);
        System.out.println("done");
        // Close the input stream.
        System.out.print("closing " + inputPath + " ... ");
        is.close();
        System.out.println("done");
        // Close the output stream.
        System.out.print("closing " + outputPath + " ... ");
        os.close();
        System.out.println("done");
    } catch (Exception e) {
        System.out.println("fail");
        e.printStackTrace();
    } finally {
        // Make sure the S3 mount point is removed.
        try {
            if (fileSystem.exists(s3Mount)) {
                System.out.print("unmounting " + s3Mount + " ... ");
                fileSystem.unmount(s3Mount);
                System.out.println("done");
            }
        } catch (Exception e) {
            System.out.println("fail");
            e.printStackTrace();
        }
        // Make sure the HDFS mount point is removed.
        try {
            if (fileSystem.exists(hdfsMount)) {
                System.out.print("unmounting " + hdfsMount + " ... ");
                fileSystem.unmount(hdfsMount);
                System.out.println("done");
            }
        } catch (Exception e) {
            System.out.println("fail");
            e.printStackTrace();
        }
    }
}