Examples with TezClient org.apache.tez.client.TezClient used on opensource projects

Example 36 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestMRRJobsDAGApi method testMultipleMRRSleepJobViaSession.

// Submits a DAG to AM via RPC after AM has started
@Test(timeout = 120000)
public void testMultipleMRRSleepJobViaSession() throws IOException, InterruptedException, TezException, ClassNotFoundException, YarnException {
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(new Random().nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
    TezClient tezSession = TezClient.create("testsession", tezConf, true);
    tezSession.start();
    Assert.assertEquals(TezAppMasterStatus.INITIALIZING, tezSession.getAppMasterStatus());
    State finalState = testMRRSleepJobDagSubmitCore(true, false, false, tezSession, false, null, null);
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, finalState);
    Assert.assertEquals(TezAppMasterStatus.READY, tezSession.getAppMasterStatus());
    finalState = testMRRSleepJobDagSubmitCore(true, false, false, tezSession, false, null, null);
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, finalState);
    Assert.assertEquals(TezAppMasterStatus.READY, tezSession.getAppMasterStatus());
    stopAndVerifyYarnApp(tezSession);
}

Example 37 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestMRRJobsDAGApi method testAMRelocalization.

// Submit 2 jobs via RPC using a custom initializer. The second job is submitted with an
// additional local resource, which is verified by the initializer.
@Test(timeout = 120000)
public void testAMRelocalization() throws Exception {
    Path relocPath = new Path("/tmp/relocalizationfilefound");
    if (remoteFs.exists(relocPath)) {
        remoteFs.delete(relocPath, true);
    }
    TezClient tezSession = createTezSession();
    State finalState = testMRRSleepJobDagSubmitCore(true, false, false, tezSession, true, MRInputAMSplitGeneratorRelocalizationTest.class, null);
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, finalState);
    Assert.assertFalse(remoteFs.exists(new Path("/tmp/relocalizationfilefound")));
    // Start the second job with some additional resources.
    // Create a test jar directly to HDFS
    LOG.info("Creating jar for relocalization test");
    Path relocFilePath = new Path("/tmp/test.jar");
    relocFilePath = remoteFs.makeQualified(relocFilePath);
    OutputStream os = remoteFs.create(relocFilePath, true);
    createTestJar(os, RELOCALIZATION_TEST_CLASS_NAME);
    // Also upload one of Tez's own JARs to HDFS and add as resource; should be ignored
    Path tezAppJar = new Path(MiniTezCluster.APPJAR);
    Path tezAppJarRemote = remoteFs.makeQualified(new Path("/tmp/" + tezAppJar.getName()));
    remoteFs.copyFromLocalFile(tezAppJar, tezAppJarRemote);
    Map<String, LocalResource> additionalResources = new HashMap<String, LocalResource>();
    additionalResources.put("test.jar", createLrObjFromPath(relocFilePath));
    additionalResources.put("TezAppJar.jar", createLrObjFromPath(tezAppJarRemote));
    Assert.assertEquals(TezAppMasterStatus.READY, tezSession.getAppMasterStatus());
    finalState = testMRRSleepJobDagSubmitCore(true, false, false, tezSession, true, MRInputAMSplitGeneratorRelocalizationTest.class, additionalResources);
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, finalState);
    Assert.assertEquals(TezAppMasterStatus.READY, tezSession.getAppMasterStatus());
    Assert.assertTrue(remoteFs.exists(new Path("/tmp/relocalizationfilefound")));
    stopAndVerifyYarnApp(tezSession);
}

Example 38 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestDAGRecovery2 method testSessionDisableMultiAttempts.

@Test(timeout = 120000)
public void testSessionDisableMultiAttempts() throws Exception {
    tezSession.stop();
    Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String.valueOf(new Random().nextInt(100000))));
    TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);
    TezConfiguration tezConf = createSessionConfig(remoteStagingDir);
    tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
    tezConf.setBoolean(TezConfiguration.DAG_RECOVERY_ENABLED, false);
    TezClient session = TezClient.create("TestDAGRecovery2SingleAttemptOnly", tezConf);
    session.start();
    // DAG should fail as it never completes on the first attempt
    DAG dag = MultiAttemptDAG.createDAG("TestSingleAttemptDAG", null);
    runDAGAndVerify(dag, State.FAILED, session);
    session.stop();
}

Example 39 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestLocalMode method testNoSysExitOnFailinglDAG.

@Test(timeout = 20000)
public void testNoSysExitOnFailinglDAG() throws TezException, InterruptedException, IOException {
    TezConfiguration tezConf1 = new TezConfiguration();
    tezConf1.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
    tezConf1.set("fs.defaultFS", "file:///");
    tezConf1.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
    // Run in non-session mode so that the AM terminates
    TezClient tezClient1 = TezClient.create("commonName", tezConf1, false);
    tezClient1.start();
    DAG dag1 = createSimpleDAG("dag1", FailingProcessor.class.getName());
    DAGClient dagClient1 = tezClient1.submitDAG(dag1);
    dagClient1.waitForCompletion();
    assertEquals(DAGStatus.State.FAILED, dagClient1.getDAGStatus(null).getState());
    // Sleep for more time than is required for the DAG to complete.
    Thread.sleep((long) (TezConstants.TEZ_DAG_SLEEP_TIME_BEFORE_EXIT * 1.5));
    dagClient1.close();
    tezClient1.stop();
}

Example 40 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestLocalMode method testMultipleClientsWithSession.

@Test(timeout = 30000)
public void testMultipleClientsWithSession() throws TezException, InterruptedException, IOException {
    TezConfiguration tezConf1 = new TezConfiguration();
    tezConf1.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
    tezConf1.set("fs.defaultFS", "file:///");
    tezConf1.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
    TezClient tezClient1 = TezClient.create("commonName", tezConf1, true);
    tezClient1.start();
    DAG dag1 = createSimpleDAG("dag1", SleepProcessor.class.getName());
    DAGClient dagClient1 = tezClient1.submitDAG(dag1);
    dagClient1.waitForCompletion();
    assertEquals(DAGStatus.State.SUCCEEDED, dagClient1.getDAGStatus(null).getState());
    dagClient1.close();
    tezClient1.stop();
    TezConfiguration tezConf2 = new TezConfiguration();
    tezConf2.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
    tezConf2.set("fs.defaultFS", "file:///");
    tezConf2.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
    DAG dag2 = createSimpleDAG("dag2", SleepProcessor.class.getName());
    TezClient tezClient2 = TezClient.create("commonName", tezConf2, true);
    tezClient2.start();
    DAGClient dagClient2 = tezClient2.submitDAG(dag2);
    dagClient2.waitForCompletion();
    assertEquals(DAGStatus.State.SUCCEEDED, dagClient2.getDAGStatus(null).getState());
    assertFalse(dagClient1.getExecutionContext().equals(dagClient2.getExecutionContext()));
    dagClient2.close();
    tezClient2.stop();
}