Examples with CommandLineJob alluxio.job.CommandLineJob used on opensource projects

Example 11 with CommandLineJob

use of alluxio.job.CommandLineJob in project alluxio by Alluxio.

the class LineageMasterIntegrationTest method lineageRecovery.

/**
   * Tests that a lineage job is executed when the output file for the lineage is reported as lost.
   *
   * The checkpoint interval is set high so that we are guaranteed to call reportLostFile
   * before persistence is complete.
   */
@Test(timeout = 100000)
@LocalAlluxioClusterResource.Config(confParams = { PropertyKey.Name.MASTER_LINEAGE_CHECKPOINT_INTERVAL_MS, "100000" })
public void lineageRecovery() throws Exception {
    final File logFile = mFolder.newFile();
    // Delete the log file so that when it starts to exist we know that it was created by the
    // lineage recompute job
    logFile.delete();
    FileSystem fs = FileSystem.Factory.get();
    try (LineageMasterClient lineageClient = getLineageMasterClient()) {
        lineageClient.createLineage(ImmutableList.<String>of(), ImmutableList.of("/testFile"), new CommandLineJob("echo hello world", new JobConf(logFile.getAbsolutePath())));
        FileOutStream out = fs.createFile(new AlluxioURI("/testFile"));
        out.write("foo".getBytes());
        out.close();
        lineageClient.reportLostFile("/testFile");
    }
    // Wait for the log file to be created by the recompute job
    CommonUtils.waitFor("the log file to be written", new Function<Void, Boolean>() {

        @Override
        public Boolean apply(Void input) {
            if (!logFile.exists()) {
                return false;
            }
            try (BufferedReader reader = new BufferedReader(new FileReader(logFile))) {
                String line = reader.readLine();
                return line != null && line.equals("hello world");
            } catch (Exception e) {
                throw Throwables.propagate(e);
            }
        }
    }, WaitForOptions.defaults().setTimeout(100 * Constants.SECOND_MS));
}

Example 12 with CommandLineJob

use of alluxio.job.CommandLineJob in project alluxio by Alluxio.

the class LineageMasterIntegrationTest method before.

@Before
public void before() throws Exception {
    AuthenticatedClientUser.set("test");
    mJob = new CommandLineJob("test", new JobConf("output"));
}

Example 13 with CommandLineJob

use of alluxio.job.CommandLineJob in project alluxio by Alluxio.

the class AlluxioLineageTest method createLineage.

@Test
public void createLineage() throws Exception {
    List<AlluxioURI> inputFiles = Lists.newArrayList(new AlluxioURI("input"));
    List<AlluxioURI> outputFiles = Lists.newArrayList(new AlluxioURI("output"));
    CommandLineJob job = new CommandLineJob("cmd", new JobConf("out"));
    mAlluxioLineage.createLineage(inputFiles, outputFiles, job);
    Mockito.verify(mLineageMasterClient).createLineage(Lists.newArrayList("input"), Lists.newArrayList("output"), job);
    // verify client is released
    Mockito.verify(mLineageContext).releaseMasterClient(mLineageMasterClient);
}