Examples with FileSystem alluxio.client.file.FileSystem used on opensource projects

Example 1 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class FileSystemMasterClientIntegrationTest method masterUnavailable.

@Test(timeout = 300000)
public void masterUnavailable() throws Exception {
    FileSystem fileSystem = mLocalAlluxioClusterResource.get().getClient();
    mLocalAlluxioClusterResource.get().getMaster().stop();
    Thread thread = new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                Thread.sleep(3000);
                mLocalAlluxioClusterResource.get().getMaster().start();
            } catch (InterruptedException e) {
                throw Throwables.propagate(e);
            }
        }
    });
    thread.start();
    fileSystem.listStatus(new AlluxioURI("/"));
    thread.join();
}

Example 2 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class ClusterInitializationTest method startCluster.

/**
   * When a user starts a new cluster, an empty root dir is created and owned by the user.
   */
@Test
@LocalAlluxioClusterResource.Config(confParams = { PropertyKey.Name.SECURITY_LOGIN_USERNAME, SUPER_USER })
public void startCluster() throws Exception {
    FileSystem fs = mLocalAlluxioClusterResource.get().getClient();
    URIStatus status = fs.getStatus(ROOT);
    Assert.assertEquals(SUPER_USER, status.getOwner());
    Assert.assertEquals(0755, status.getMode());
    Assert.assertEquals(0, fs.listStatus(new AlluxioURI("/")).size());
}

Example 3 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class LineageMasterIntegrationTest method docExample.

/**
   * Runs code given in the docs (http://alluxio.org/documentation/Lineage-API.html) to make sure it
   * actually runs.
   *
   * If you need to update the doc-code here, make sure you also update it in the docs.
   */
@Test
public void docExample() throws Exception {
    // create input files
    FileSystem fs = FileSystem.Factory.get();
    fs.createFile(new AlluxioURI("/inputFile1")).close();
    fs.createFile(new AlluxioURI("/inputFile2")).close();
    // ------ code block from docs ------
    AlluxioLineage tl = AlluxioLineage.get();
    // input file paths
    AlluxioURI input1 = new AlluxioURI("/inputFile1");
    AlluxioURI input2 = new AlluxioURI("/inputFile2");
    ArrayList<AlluxioURI> inputFiles = new ArrayList<>();
    Collections.addAll(inputFiles, input1, input2);
    // output file paths
    AlluxioURI output = new AlluxioURI("/outputFile");
    ArrayList<AlluxioURI> outputFiles = new ArrayList<>();
    Collections.addAll(outputFiles, output);
    // command-line job
    JobConf conf = new JobConf("/tmp/recompute.log");
    CommandLineJob job = new CommandLineJob("my-spark-job.sh", conf);
    long lineageId = tl.createLineage(inputFiles, outputFiles, job);
    // ------ code block from docs ------
    DeleteLineageOptions options = DeleteLineageOptions.defaults().setCascade(true);
    tl.deleteLineage(lineageId);
    fs.delete(new AlluxioURI("/outputFile"));
    lineageId = tl.createLineage(inputFiles, outputFiles, job);
    // ------ code block from docs ------
    tl.deleteLineage(lineageId, options);
}

Example 4 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class MiniBenchmark method writeFile.

/**
   * Writes a file.
   *
   * @param fileSize the file size
   * @param iterations number of iterations
   * @throws Exception it if fails to write
   */
private static void writeFile(long fileSize, int iterations) throws Exception {
    FileSystem fileSystem = FileSystem.Factory.get();
    byte[] buffer = new byte[(int) Math.min(fileSize, 4 * Constants.MB)];
    Arrays.fill(buffer, (byte) 'a');
    AlluxioURI path = new AlluxioURI(TEST_PATH);
    long runTime = 0;
    for (int i = 0; i < iterations; i++) {
        if (fileSystem.exists(path)) {
            fileSystem.delete(path);
        }
        long bytesWritten = 0;
        long start = System.nanoTime();
        try (FileOutStream outStream = fileSystem.createFile(new AlluxioURI(TEST_PATH))) {
            while (bytesWritten < fileSize) {
                outStream.write(buffer, 0, (int) Math.min(buffer.length, fileSize - bytesWritten));
                bytesWritten += buffer.length;
            }
        }
        runTime += System.nanoTime() - start;
    }
    System.out.printf("Runtime: %f seconds.%n", runTime * 1.0 / Constants.SECOND_NANO);
}

Example 5 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class UnderStorageSystemInterfaceIntegrationTest method objectLoadMetadata.

/**
   * Tests load metadata on list with an object store having pre-populated pseudo-directories.
   */
@Test
public void objectLoadMetadata() throws Exception {
    // Only run test for an object store
    Assume.assumeTrue(UnderFileSystemUtils.isObjectStorage(mUfs));
    ObjectUnderFileSystem ufs = (ObjectUnderFileSystem) mUfs.getUnderFileSystem();
    ObjectStorePreConfig config = prepareObjectStore(ufs);
    String baseDirectoryName = config.getBaseDirectoryPath().substring(PathUtils.normalizePath(mUnderfsAddress, "/").length());
    AlluxioURI rootAlluxioURI = new AlluxioURI(PathUtils.concatPath("/", baseDirectoryName));
    FileSystem client = mLocalAlluxioClusterResource.get().getClient();
    List<URIStatus> results = client.listStatus(rootAlluxioURI, ListStatusOptions.defaults().setLoadMetadataType(LoadMetadataType.Always));
    Assert.assertEquals(config.getSubDirectoryNames().length + config.getFileNames().length, results.size());
}