Examples with SimpleSessionExample org.apache.tez.examples.SimpleSessionExample used on opensource projects

Example 1 with SimpleSessionExample

use of org.apache.tez.examples.SimpleSessionExample in project tez by apache.

the class TestTezJobs method testInvalidQueueSubmission.

@Test(timeout = 60000)
public void testInvalidQueueSubmission() throws Exception {
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    YarnClient yarnClient = YarnClient.createYarnClient();
    try {
        yarnClient.init(mrrTezCluster.getConfig());
        yarnClient.start();
        SimpleSessionExample job = new SimpleSessionExample();
        tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
        tezConf.set(TezConfiguration.TEZ_QUEUE_NAME, "nonexistent");
        String[] inputPaths = new String[1];
        String[] outputPaths = new String[1];
        String inputDirStr = "/tmp/owc-input";
        inputPaths[0] = inputDirStr;
        Path inputDir = new Path(inputDirStr);
        remoteFs.mkdirs(inputDir);
        String outputDirStr = "/tmp/owc-output";
        outputPaths[0] = outputDirStr;
        int result = job.run(tezConf, new String[] { StringUtils.join(",", inputPaths), StringUtils.join(",", outputPaths), "2" }, null);
        Assert.assertTrue("Job should have failed", result != 0);
    } catch (TezException e) {
        Assert.assertTrue(e.getMessage().contains("Failed to submit application"));
    } finally {
        if (yarnClient != null) {
            yarnClient.stop();
        }
    }
}

Example 2 with SimpleSessionExample

use of org.apache.tez.examples.SimpleSessionExample in project tez by apache.

the class TestTezJobs method testInvalidQueueSubmissionToSession.

@Test(timeout = 60000)
public void testInvalidQueueSubmissionToSession() throws Exception {
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    YarnClient yarnClient = YarnClient.createYarnClient();
    try {
        yarnClient.init(mrrTezCluster.getConfig());
        yarnClient.start();
        SimpleSessionExample job = new SimpleSessionExample();
        tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
        tezConf.set(TezConfiguration.TEZ_QUEUE_NAME, "nonexistent");
        String[] inputPaths = new String[1];
        String[] outputPaths = new String[1];
        String inputDirStr = "/tmp/owc-input";
        inputPaths[0] = inputDirStr;
        Path inputDir = new Path(inputDirStr);
        remoteFs.mkdirs(inputDir);
        String outputDirStr = "/tmp/owc-output";
        outputPaths[0] = outputDirStr;
        job.run(tezConf, new String[] { StringUtils.join(",", inputPaths), StringUtils.join(",", outputPaths), "2" }, null);
        fail("Job submission should have failed");
    } catch (SessionNotRunning e) {
        // Expected
        LOG.info("Session not running", e);
    } catch (TezException e) {
        Assert.assertTrue(e.getMessage().contains("Failed to submit application"));
    } finally {
        if (yarnClient != null) {
            yarnClient.stop();
        }
    }
}

Example 3 with SimpleSessionExample

use of org.apache.tez.examples.SimpleSessionExample in project tez by apache.

the class TestTezJobs method testSimpleSessionExample.

@Test(timeout = 60000)
public void testSimpleSessionExample() throws Exception {
    Path stagingDirPath = new Path("/tmp/owc-staging-dir");
    remoteFs.mkdirs(stagingDirPath);
    int numIterations = 2;
    String[] inputPaths = new String[numIterations];
    String[] outputPaths = new String[numIterations];
    Path[] outputDirs = new Path[numIterations];
    for (int i = 0; i < numIterations; ++i) {
        String inputDirStr = "/tmp/owc-input-" + i + "/";
        inputPaths[i] = inputDirStr;
        Path inputDir = new Path(inputDirStr);
        remoteFs.mkdirs(inputDir);
        generateOrderedWordCountInput(inputDir, remoteFs);
        String outputDirStr = "/tmp/owc-output-" + i + "/";
        outputPaths[i] = outputDirStr;
        Path outputDir = new Path(outputDirStr);
        outputDirs[i] = outputDir;
    }
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
    YarnClient yarnClient = YarnClient.createYarnClient();
    try {
        yarnClient.init(mrrTezCluster.getConfig());
        yarnClient.start();
        List<ApplicationReport> apps = yarnClient.getApplications();
        int appsBeforeCount = apps != null ? apps.size() : 0;
        SimpleSessionExample job = new SimpleSessionExample();
        tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
        Assert.assertTrue("SimpleSessionExample failed", job.run(tezConf, new String[] { StringUtils.join(",", inputPaths), StringUtils.join(",", outputPaths), "2" }, null) == 0);
        for (int i = 0; i < numIterations; ++i) {
            verifyOutput(outputDirs[i], remoteFs);
        }
        apps = yarnClient.getApplications();
        int appsAfterCount = apps != null ? apps.size() : 0;
        // Running in session mode. So should only create 1 more app.
        Assert.assertEquals(appsBeforeCount + 1, appsAfterCount);
    } finally {
        remoteFs.delete(stagingDirPath, true);
        if (yarnClient != null) {
            yarnClient.stop();
        }
    }
}