Examples with CommandLineJob alluxio.job.CommandLineJob used on opensource projects

Example 6 with CommandLineJob

use of alluxio.job.CommandLineJob in project alluxio by Alluxio.

the class LineageMasterIntegrationTest method docExample.

/**
   * Runs code given in the docs (http://alluxio.org/documentation/Lineage-API.html) to make sure it
   * actually runs.
   *
   * If you need to update the doc-code here, make sure you also update it in the docs.
   */
@Test
public void docExample() throws Exception {
    // create input files
    FileSystem fs = FileSystem.Factory.get();
    fs.createFile(new AlluxioURI("/inputFile1")).close();
    fs.createFile(new AlluxioURI("/inputFile2")).close();
    // ------ code block from docs ------
    AlluxioLineage tl = AlluxioLineage.get();
    // input file paths
    AlluxioURI input1 = new AlluxioURI("/inputFile1");
    AlluxioURI input2 = new AlluxioURI("/inputFile2");
    ArrayList<AlluxioURI> inputFiles = new ArrayList<>();
    Collections.addAll(inputFiles, input1, input2);
    // output file paths
    AlluxioURI output = new AlluxioURI("/outputFile");
    ArrayList<AlluxioURI> outputFiles = new ArrayList<>();
    Collections.addAll(outputFiles, output);
    // command-line job
    JobConf conf = new JobConf("/tmp/recompute.log");
    CommandLineJob job = new CommandLineJob("my-spark-job.sh", conf);
    long lineageId = tl.createLineage(inputFiles, outputFiles, job);
    // ------ code block from docs ------
    DeleteLineageOptions options = DeleteLineageOptions.defaults().setCascade(true);
    tl.deleteLineage(lineageId);
    fs.delete(new AlluxioURI("/outputFile"));
    lineageId = tl.createLineage(inputFiles, outputFiles, job);
    // ------ code block from docs ------
    tl.deleteLineage(lineageId, options);
}

Example 7 with CommandLineJob

use of alluxio.job.CommandLineJob in project alluxio by Alluxio.

the class Lineage method fromJournalEntry.

/**
   * Converts the entry to a {@link Lineage}.
   *
   * @param entry the entry to convert
   * @return the {@link Lineage} representation
   */
public static Lineage fromJournalEntry(LineageEntry entry) {
    List<Long> inputFiles = new ArrayList<>(entry.getInputFilesList());
    List<Long> outputFiles = new ArrayList<>(entry.getOutputFileIdsList());
    Job job = new CommandLineJob(entry.getJobCommand(), new JobConf(entry.getJobOutputPath()));
    return new Lineage(entry.getId(), inputFiles, outputFiles, job, entry.getCreationTimeMs());
}

Example 8 with CommandLineJob

use of alluxio.job.CommandLineJob in project alluxio by Alluxio.

the class LineageMasterTest method before.

/**
   * Sets up all dependencies before a test runs.
   */
@Before
public void before() throws Exception {
    JournalFactory journalFactory = new JournalFactory.ReadWrite(mTestFolder.newFolder().getAbsolutePath());
    mFileSystemMaster = Mockito.mock(FileSystemMaster.class);
    ThreadFactory threadPool = ThreadFactoryUtils.build("LineageMasterTest-%d", true);
    mExecutorService = Executors.newFixedThreadPool(2, threadPool);
    mLineageMaster = new LineageMaster(mFileSystemMaster, journalFactory, ExecutorServiceFactories.constantExecutorServiceFactory(mExecutorService));
    mJob = new CommandLineJob("test", new JobConf("output"));
}

Example 9 with CommandLineJob

use of alluxio.job.CommandLineJob in project alluxio by Alluxio.

the class LineageStoreTest method before.

/**
   * Sets up the dependencies before a test runs.
   */
@Before
public void before() {
    mLineageStore = new LineageStore(new LineageIdGenerator());
    mJob = new CommandLineJob("test", new JobConf("output"));
}

Example 10 with CommandLineJob

use of alluxio.job.CommandLineJob in project alluxio by Alluxio.

the class LineageMasterClientRestApiTest method getLineageInfoList.

@Test
@Config(confParams = { PropertyKey.Name.USER_LINEAGE_ENABLED, "true" })
public void getLineageInfoList() throws Exception {
    AlluxioURI input = new AlluxioURI("/input");
    AlluxioURI output = new AlluxioURI("/output");
    LineageMaster lineageMaster = mMaster.getLineageMaster();
    mLineageClient.createFile(new AlluxioURI("/input")).close();
    long lineageId = lineageMaster.createLineage(Lists.newArrayList(input), Lists.newArrayList(output), new CommandLineJob("test", new JobConf(output.getPath())));
    String result = new TestCase(mHostname, mPort, getEndpoint(LineageMasterClientRestServiceHandler.GET_LINEAGE_INFO_LIST), NO_PARAMS, HttpMethod.GET, null).call();
    List<LineageInfo> lineageInfos = new ObjectMapper().readValue(result, new TypeReference<List<LineageInfo>>() {
    });
    LineageInfo lineageInfo = Iterables.getOnlyElement(lineageInfos);
    Assert.assertEquals(ImmutableList.of(input.toString()), lineageInfo.getInputFiles());
    Assert.assertEquals(ImmutableList.of(output.toString()), lineageInfo.getOutputFiles());
    Assert.assertEquals(lineageId, lineageInfo.getId());
}