Examples with MultipleCommitsExample org.apache.tez.mapreduce.examples.MultipleCommitsExample used on opensource projects

Example 1 with MultipleCommitsExample

use of org.apache.tez.mapreduce.examples.MultipleCommitsExample in project tez by apache.

the class TestTezJobs method testMultipleCommits_OnVertexSuccess.

@Test(timeout = 60000)
public void testMultipleCommits_OnVertexSuccess() throws Exception {
    Path stagingDirPath = new Path("/tmp/commit-staging-dir");
    Random rand = new Random();
    String v1OutputPathPrefix = "/tmp/commit-output-v1";
    int v1OutputNum = rand.nextInt(10) + 1;
    String v2OutputPathPrefix = "/tmp/commit-output-v2";
    int v2OutputNum = rand.nextInt(10) + 1;
    String uv12OutputPathPrefix = "/tmp/commit-output-uv12";
    int uv12OutputNum = rand.nextInt(10) + 1;
    String v3OutputPathPrefix = "/tmp/commit-output-v3";
    int v3OutputNum = rand.nextInt(10) + 1;
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
    TezClient tezSession = null;
    try {
        MultipleCommitsExample job = new MultipleCommitsExample();
        Assert.assertTrue("MultipleCommitsExample failed", job.run(tezConf, new String[] { v1OutputPathPrefix, v1OutputNum + "", v2OutputPathPrefix, v2OutputNum + "", uv12OutputPathPrefix, uv12OutputNum + "", v3OutputPathPrefix, v3OutputNum + "", MultipleCommitsExample.CommitOnVertexSuccessOption }, null) == 0);
        verifyCommits(v1OutputPathPrefix, v1OutputNum);
        verifyCommits(v2OutputPathPrefix, v2OutputNum);
        verifyCommits(uv12OutputPathPrefix, uv12OutputNum);
        verifyCommits(v3OutputPathPrefix, v3OutputNum);
    } finally {
        remoteFs.delete(stagingDirPath, true);
        if (tezSession != null) {
            tezSession.stop();
        }
    }
}

Example 2 with MultipleCommitsExample

use of org.apache.tez.mapreduce.examples.MultipleCommitsExample in project tez by apache.

the class TestTezJobs method testMultipleCommits_OnDAGSuccess.

@Test(timeout = 60000)
public void testMultipleCommits_OnDAGSuccess() throws Exception {
    Path stagingDirPath = new Path("/tmp/commit-staging-dir");
    Random rand = new Random();
    String v1OutputPathPrefix = "/tmp/commit-output-v1";
    int v1OutputNum = rand.nextInt(10) + 1;
    String v2OutputPathPrefix = "/tmp/commit-output-v2";
    int v2OutputNum = rand.nextInt(10) + 1;
    String uv12OutputPathPrefix = "/tmp/commit-output-uv12";
    int uv12OutputNum = rand.nextInt(10) + 1;
    String v3OutputPathPrefix = "/tmp/commit-output-v3";
    int v3OutputNum = rand.nextInt(10) + 1;
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
    TezClient tezSession = null;
    try {
        MultipleCommitsExample job = new MultipleCommitsExample();
        Assert.assertTrue("MultipleCommitsExample failed", job.run(tezConf, new String[] { v1OutputPathPrefix, v1OutputNum + "", v2OutputPathPrefix, v2OutputNum + "", uv12OutputPathPrefix, uv12OutputNum + "", v3OutputPathPrefix, v3OutputNum + "" }, null) == 0);
        verifyCommits(v1OutputPathPrefix, v1OutputNum);
        verifyCommits(v2OutputPathPrefix, v2OutputNum);
        verifyCommits(uv12OutputPathPrefix, uv12OutputNum);
        verifyCommits(v3OutputPathPrefix, v3OutputNum);
    } finally {
        remoteFs.delete(stagingDirPath, true);
        if (tezSession != null) {
            tezSession.stop();
        }
    }
}