Examples with AlluxioURI alluxio.AlluxioURI used on opensource projects

Example 66 with AlluxioURI

use of alluxio.AlluxioURI in project alluxio by Alluxio.

the class LineageMasterClientRestApiTest method deleteLineage.

@Test
@Config(confParams = { PropertyKey.Name.USER_LINEAGE_ENABLED, "true" })
public void deleteLineage() throws Exception {
    LineageMaster lineageMaster = mMaster.getLineageMaster();
    long lineageId = lineageMaster.createLineage(new ArrayList<AlluxioURI>(), new ArrayList<AlluxioURI>(), new CommandLineJob("test", new JobConf("/output")));
    Map<String, String> params = new HashMap<>();
    params.put("lineageId", Long.toString(lineageId));
    params.put("cascade", "false");
    new TestCase(mHostname, mPort, getEndpoint(LineageMasterClientRestServiceHandler.DELETE_LINEAGE), params, HttpMethod.POST, true).run();
}

Example 67 with AlluxioURI

use of alluxio.AlluxioURI in project alluxio by Alluxio.

the class LineageMasterClientRestApiTest method createLineage.

@Test
@Config(confParams = { PropertyKey.Name.USER_LINEAGE_ENABLED, "true" })
public void createLineage() throws Exception {
    mLineageClient.createFile(new AlluxioURI("/input")).close();
    Map<String, String> params = new HashMap<>();
    params.put("inputFiles", "/input");
    params.put("outputFiles", "/output");
    params.put("command", "test");
    params.put("commandOutputFile", "test");
    new TestCase(mHostname, mPort, getEndpoint(LineageMasterClientRestServiceHandler.CREATE_LINEAGE), params, HttpMethod.POST, null).call();
}

Example 68 with AlluxioURI

use of alluxio.AlluxioURI in project alluxio by Alluxio.

the class LineageMasterIntegrationTest method docExample.

/**
   * Runs code given in the docs (http://alluxio.org/documentation/Lineage-API.html) to make sure it
   * actually runs.
   *
   * If you need to update the doc-code here, make sure you also update it in the docs.
   */
@Test
public void docExample() throws Exception {
    // create input files
    FileSystem fs = FileSystem.Factory.get();
    fs.createFile(new AlluxioURI("/inputFile1")).close();
    fs.createFile(new AlluxioURI("/inputFile2")).close();
    // ------ code block from docs ------
    AlluxioLineage tl = AlluxioLineage.get();
    // input file paths
    AlluxioURI input1 = new AlluxioURI("/inputFile1");
    AlluxioURI input2 = new AlluxioURI("/inputFile2");
    ArrayList<AlluxioURI> inputFiles = new ArrayList<>();
    Collections.addAll(inputFiles, input1, input2);
    // output file paths
    AlluxioURI output = new AlluxioURI("/outputFile");
    ArrayList<AlluxioURI> outputFiles = new ArrayList<>();
    Collections.addAll(outputFiles, output);
    // command-line job
    JobConf conf = new JobConf("/tmp/recompute.log");
    CommandLineJob job = new CommandLineJob("my-spark-job.sh", conf);
    long lineageId = tl.createLineage(inputFiles, outputFiles, job);
    // ------ code block from docs ------
    DeleteLineageOptions options = DeleteLineageOptions.defaults().setCascade(true);
    tl.deleteLineage(lineageId);
    fs.delete(new AlluxioURI("/outputFile"));
    lineageId = tl.createLineage(inputFiles, outputFiles, job);
    // ------ code block from docs ------
    tl.deleteLineage(lineageId, options);
}

Example 69 with AlluxioURI

use of alluxio.AlluxioURI in project alluxio by Alluxio.

the class LineageMasterIntegrationTest method lineageCreation.

@Test
public void lineageCreation() throws Exception {
    try (LineageMasterClient lineageMasterClient = getLineageMasterClient()) {
        ArrayList<String> outFiles = new ArrayList<>();
        Collections.addAll(outFiles, OUT_FILE);
        lineageMasterClient.createLineage(new ArrayList<String>(), outFiles, mJob);
        List<LineageInfo> infos = lineageMasterClient.getLineageInfoList();
        Assert.assertEquals(1, infos.size());
        AlluxioURI uri = new AlluxioURI(infos.get(0).getOutputFiles().get(0));
        URIStatus status = getFileSystemMasterClient().getStatus(uri);
        Assert.assertEquals(PersistenceState.NOT_PERSISTED.toString(), status.getPersistenceState());
        Assert.assertFalse(status.isCompleted());
    }
}

Example 70 with AlluxioURI

use of alluxio.AlluxioURI in project alluxio by Alluxio.

the class CheckConsistencyIntegrationTest method inconsistent.

/**
   * Tests the {@link FileSystemMaster#checkConsistency(AlluxioURI, CheckConsistencyOptions)} method
   * when no files are consistent.
   */
@Test
public void inconsistent() throws Exception {
    String ufsDirectory = mFileSystem.getStatus(DIRECTORY).getUfsPath();
    UnderFileSystem ufs = UnderFileSystem.Factory.get(ufsDirectory);
    ufs.deleteDirectory(ufsDirectory, DeleteOptions.defaults().setRecursive(true));
    List<AlluxioURI> expected = Lists.newArrayList(FILE, DIRECTORY);
    List<AlluxioURI> result = mFileSystemMaster.checkConsistency(new AlluxioURI("/"), CheckConsistencyOptions.defaults());
    Collections.sort(expected);
    Collections.sort(result);
    Assert.assertEquals(expected, result);
}