Examples with JobConf alluxio.job.JobConf used on opensource projects

Example 6 with JobConf

use of alluxio.job.JobConf in project alluxio by Alluxio.

the class Lineage method fromJournalEntry.

/**
   * Converts the entry to a {@link Lineage}.
   *
   * @param entry the entry to convert
   * @return the {@link Lineage} representation
   */
public static Lineage fromJournalEntry(LineageEntry entry) {
    List<Long> inputFiles = new ArrayList<>(entry.getInputFilesList());
    List<Long> outputFiles = new ArrayList<>(entry.getOutputFileIdsList());
    Job job = new CommandLineJob(entry.getJobCommand(), new JobConf(entry.getJobOutputPath()));
    return new Lineage(entry.getId(), inputFiles, outputFiles, job, entry.getCreationTimeMs());
}

Example 7 with JobConf

use of alluxio.job.JobConf in project alluxio by Alluxio.

the class LineageMasterTest method before.

/**
   * Sets up all dependencies before a test runs.
   */
@Before
public void before() throws Exception {
    JournalFactory journalFactory = new JournalFactory.ReadWrite(mTestFolder.newFolder().getAbsolutePath());
    mFileSystemMaster = Mockito.mock(FileSystemMaster.class);
    ThreadFactory threadPool = ThreadFactoryUtils.build("LineageMasterTest-%d", true);
    mExecutorService = Executors.newFixedThreadPool(2, threadPool);
    mLineageMaster = new LineageMaster(mFileSystemMaster, journalFactory, ExecutorServiceFactories.constantExecutorServiceFactory(mExecutorService));
    mJob = new CommandLineJob("test", new JobConf("output"));
}

Example 8 with JobConf

use of alluxio.job.JobConf in project alluxio by Alluxio.

the class LineageStoreTest method before.

/**
   * Sets up the dependencies before a test runs.
   */
@Before
public void before() {
    mLineageStore = new LineageStore(new LineageIdGenerator());
    mJob = new CommandLineJob("test", new JobConf("output"));
}

Example 9 with JobConf

use of alluxio.job.JobConf in project alluxio by Alluxio.

the class LineageMasterClientRestApiTest method getLineageInfoList.

@Test
@Config(confParams = { PropertyKey.Name.USER_LINEAGE_ENABLED, "true" })
public void getLineageInfoList() throws Exception {
    AlluxioURI input = new AlluxioURI("/input");
    AlluxioURI output = new AlluxioURI("/output");
    LineageMaster lineageMaster = mMaster.getLineageMaster();
    mLineageClient.createFile(new AlluxioURI("/input")).close();
    long lineageId = lineageMaster.createLineage(Lists.newArrayList(input), Lists.newArrayList(output), new CommandLineJob("test", new JobConf(output.getPath())));
    String result = new TestCase(mHostname, mPort, getEndpoint(LineageMasterClientRestServiceHandler.GET_LINEAGE_INFO_LIST), NO_PARAMS, HttpMethod.GET, null).call();
    List<LineageInfo> lineageInfos = new ObjectMapper().readValue(result, new TypeReference<List<LineageInfo>>() {
    });
    LineageInfo lineageInfo = Iterables.getOnlyElement(lineageInfos);
    Assert.assertEquals(ImmutableList.of(input.toString()), lineageInfo.getInputFiles());
    Assert.assertEquals(ImmutableList.of(output.toString()), lineageInfo.getOutputFiles());
    Assert.assertEquals(lineageId, lineageInfo.getId());
}

Example 10 with JobConf

use of alluxio.job.JobConf in project alluxio by Alluxio.

the class LineageMasterIntegrationTest method lineageRecovery.

/**
   * Tests that a lineage job is executed when the output file for the lineage is reported as lost.
   *
   * The checkpoint interval is set high so that we are guaranteed to call reportLostFile
   * before persistence is complete.
   */
@Test(timeout = 100000)
@LocalAlluxioClusterResource.Config(confParams = { PropertyKey.Name.MASTER_LINEAGE_CHECKPOINT_INTERVAL_MS, "100000" })
public void lineageRecovery() throws Exception {
    final File logFile = mFolder.newFile();
    // Delete the log file so that when it starts to exist we know that it was created by the
    // lineage recompute job
    logFile.delete();
    FileSystem fs = FileSystem.Factory.get();
    try (LineageMasterClient lineageClient = getLineageMasterClient()) {
        lineageClient.createLineage(ImmutableList.<String>of(), ImmutableList.of("/testFile"), new CommandLineJob("echo hello world", new JobConf(logFile.getAbsolutePath())));
        FileOutStream out = fs.createFile(new AlluxioURI("/testFile"));
        out.write("foo".getBytes());
        out.close();
        lineageClient.reportLostFile("/testFile");
    }
    // Wait for the log file to be created by the recompute job
    CommonUtils.waitFor("the log file to be written", new Function<Void, Boolean>() {

        @Override
        public Boolean apply(Void input) {
            if (!logFile.exists()) {
                return false;
            }
            try (BufferedReader reader = new BufferedReader(new FileReader(logFile))) {
                String line = reader.readLine();
                return line != null && line.equals("hello world");
            } catch (Exception e) {
                throw Throwables.propagate(e);
            }
        }
    }, WaitForOptions.defaults().setTimeout(100 * Constants.SECOND_MS));
}