Examples with DAGStatus org.apache.tez.dag.api.client.DAGStatus used on opensource projects

Example 16 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestDAGRecovery method runDAGAndVerify.

void runDAGAndVerify(DAG dag, DAGStatus.State finalState) throws Exception {
    tezSession.waitTillReady();
    DAGClient dagClient = tezSession.submitDAG(dag);
    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
        LOG.info("Waiting for dag to complete. Sleeping for 500ms." + " DAG name: " + dag.getName() + " DAG appContext: " + dagClient.getExecutionContext() + " Current state: " + dagStatus.getState());
        Thread.sleep(100);
        dagStatus = dagClient.getDAGStatus(null);
    }
    Assert.assertEquals(finalState, dagStatus.getState());
}

Example 17 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestExceptionPropagation method testExceptionPropagationSession.

/**
 * verify the diagnostics in DAGStatus is correct in session mode, using local
 * mode for fast speed
 *
 * @throws Exception
 */
@Test(timeout = 600000)
public void testExceptionPropagationSession() throws Exception {
    try {
        startSessionClient();
        for (ExceptionLocation exLocation : ExceptionLocation.values()) {
            LOG.info("Session mode, Test for Exception from:" + exLocation.name());
            DAG dag = createDAG(exLocation);
            DAGClient dagClient = tezSession.submitDAG(dag);
            DAGStatus dagStatus = dagClient.waitForCompletion();
            String diagnostics = StringUtils.join(dagStatus.getDiagnostics(), ",");
            LOG.info("Diagnostics:" + diagnostics);
            if (exLocation == ExceptionLocation.PROCESSOR_COUNTER_EXCEEDED) {
                assertTrue(diagnostics.contains("Too many counters"));
            } else {
                assertTrue(diagnostics.contains(exLocation.name()));
            }
        }
    } finally {
        stopSessionClient();
    }
}

Example 18 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestLocalMode method testMultiDAGsOnSession.

@Test(timeout = 30000)
public void testMultiDAGsOnSession() throws IOException, TezException, InterruptedException {
    // two dags will be submitted to session
    int dags = 2;
    String[] inputPaths = new String[dags];
    String[] outputPaths = new String[dags];
    DAGClient[] dagClients = new DAGClient[dags];
    TezConfiguration tezConf = new TezConfiguration();
    tezConf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
    tezConf.set("fs.defaultFS", "file:///");
    tezConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
    TezClient tezClient = TezClient.create("testMultiDAGOnSession", tezConf, true);
    tezClient.start();
    // create inputs and outputs
    FileSystem fs = FileSystem.get(tezConf);
    for (int i = 0; i < dags; i++) {
        inputPaths[i] = new Path(TEST_DIR.getAbsolutePath(), "in-" + i).toString();
        createInputFile(fs, inputPaths[i]);
        outputPaths[i] = new Path(TEST_DIR.getAbsolutePath(), "out-" + i).toString();
    }
    // start testing
    try {
        for (int i = 0; i < inputPaths.length; ++i) {
            DAG dag = OrderedWordCount.createDAG(tezConf, inputPaths[i], outputPaths[i], 1, false, false, // the names of the DAGs must be unique in a session
            ("DAG-Iteration-" + i));
            tezClient.waitTillReady();
            System.out.println("Running dag number " + i);
            dagClients[i] = tezClient.submitDAG(dag);
            // wait to finish
            DAGStatus dagStatus = dagClients[i].waitForCompletion();
            if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
                fail("Iteration " + i + " failed with diagnostics: " + dagStatus.getDiagnostics());
            }
            // verify all dags sharing the same execution context
            if (i > 0) {
                assertTrue(dagClients[i - 1].getExecutionContext().equals(dagClients[i].getExecutionContext()));
            }
        }
    } finally {
        tezClient.stop();
    }
}

Example 19 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestMRRJobsDAGApi method testSleepJob.

@Test(timeout = 60000)
public void testSleepJob() throws TezException, IOException, InterruptedException {
    SleepProcessorConfig spConf = new SleepProcessorConfig(1);
    DAG dag = DAG.create("TezSleepProcessor");
    Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1, Resource.newInstance(1024, 1));
    dag.addVertex(vertex);
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
    TezClient tezSession = TezClient.create("TezSleepProcessor", tezConf, false);
    tezSession.start();
    DAGClient dagClient = tezSession.submitDAG(dag);
    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
        LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: " + dagStatus.getState());
        Thread.sleep(500l);
        dagStatus = dagClient.getDAGStatus(null);
    }
    dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
    assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
    assertNotNull(dagStatus.getDAGCounters());
    assertNotNull(dagStatus.getDAGCounters().getGroup(FileSystemCounter.class.getName()));
    assertNotNull(dagStatus.getDAGCounters().findCounter(TaskCounter.GC_TIME_MILLIS));
    ExampleDriver.printDAGStatus(dagClient, new String[] { "SleepVertex" }, true, true);
    tezSession.stop();
}

Example 20 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestMRRJobsDAGApi method testNonDefaultFSStagingDir.

@Test(timeout = 60000)
public void testNonDefaultFSStagingDir() throws Exception {
    SleepProcessorConfig spConf = new SleepProcessorConfig(1);
    DAG dag = DAG.create("TezSleepProcessor");
    Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1, Resource.newInstance(1024, 1));
    dag.addVertex(vertex);
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    Path stagingDir = new Path(TEST_ROOT_DIR, "testNonDefaultFSStagingDir" + String.valueOf(random.nextInt(100000)));
    FileSystem localFs = FileSystem.getLocal(tezConf);
    stagingDir = localFs.makeQualified(stagingDir);
    localFs.mkdirs(stagingDir);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());
    TezClient tezSession = TezClient.create("TezSleepProcessor", tezConf, false);
    tezSession.start();
    DAGClient dagClient = tezSession.submitDAG(dag);
    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
        LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: " + dagStatus.getState());
        Thread.sleep(500l);
        dagStatus = dagClient.getDAGStatus(null);
    }
    dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
    assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
    assertNotNull(dagStatus.getDAGCounters());
    assertNotNull(dagStatus.getDAGCounters().getGroup(FileSystemCounter.class.getName()));
    assertNotNull(dagStatus.getDAGCounters().findCounter(TaskCounter.GC_TIME_MILLIS));
    ExampleDriver.printDAGStatus(dagClient, new String[] { "SleepVertex" }, true, true);
    tezSession.stop();
}