Examples with DAGStatus org.apache.tez.dag.api.client.DAGStatus used on opensource projects

Example 31 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestExternalTezServices method runExceptionSimulation.

private void runExceptionSimulation() throws IOException, TezException, InterruptedException {
    DAG dag = DAG.create(ContainerRunnerImpl.DAG_NAME_INSTRUMENTED_FAILURES);
    Vertex v = Vertex.create("Vertex1", ProcessorDescriptor.create(SleepProcessor.class.getName()), 3);
    v.setExecutionContext(EXECUTION_CONTEXT_EXT_SERVICE_PUSH);
    dag.addVertex(v);
    DAGClient dagClient = extServiceTestHelper.getSharedTezClient().submitDAG(dag);
    DAGStatus dagStatus = dagClient.waitForCompletion();
    assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
    assertEquals(1, dagStatus.getDAGProgress().getFailedTaskAttemptCount());
    assertEquals(1, dagStatus.getDAGProgress().getRejectedTaskAttemptCount());
}

Example 32 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestATSHistoryWithMiniCluster method testDisabledACls.

@Test(timeout = 50000)
public void testDisabledACls() throws Exception {
    TezClient tezSession = null;
    try {
        SleepProcessorConfig spConf = new SleepProcessorConfig(1);
        DAG dag = DAG.create("TezSleepProcessor");
        Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1, Resource.newInstance(256, 1));
        dag.addVertex(vertex);
        TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
        tezConf.setBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS, true);
        tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName());
        Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
        remoteFs.mkdirs(remoteStagingDir);
        tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
        tezSession = TezClient.create("TezSleepProcessor", tezConf, true);
        tezSession.start();
        DAGClient dagClient = tezSession.submitDAG(dag);
        DAGStatus dagStatus = dagClient.getDAGStatus(null);
        while (!dagStatus.isCompleted()) {
            LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: " + dagStatus.getState());
            Thread.sleep(500l);
            dagStatus = dagClient.getDAGStatus(null);
        }
        Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
    } finally {
        if (tezSession != null) {
            tezSession.stop();
        }
    }
}

Example 33 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestATSHistoryWithACLs method testSimpleAMACls.

@Test(timeout = 50000)
public void testSimpleAMACls() throws Exception {
    TezClient tezSession = null;
    ApplicationId applicationId;
    String viewAcls = "nobody nobody_group";
    try {
        SleepProcessorConfig spConf = new SleepProcessorConfig(1);
        DAG dag = DAG.create("TezSleepProcessor");
        Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1, Resource.newInstance(256, 1));
        dag.addVertex(vertex);
        TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
        tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
        tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName());
        Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
        remoteFs.mkdirs(remoteStagingDir);
        tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
        tezSession = TezClient.create("TezSleepProcessor", tezConf, true);
        tezSession.start();
        applicationId = tezSession.getAppMasterApplicationId();
        DAGClient dagClient = tezSession.submitDAG(dag);
        DAGStatus dagStatus = dagClient.getDAGStatus(null);
        while (!dagStatus.isCompleted()) {
            LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: " + dagStatus.getState());
            Thread.sleep(500l);
            dagStatus = dagClient.getDAGStatus(null);
        }
        assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
    } finally {
        if (tezSession != null) {
            tezSession.stop();
        }
    }
    TimelineDomain timelineDomain = getDomain(ATSHistoryACLPolicyManager.DOMAIN_ID_PREFIX + applicationId.toString());
    verifyDomainACLs(timelineDomain, Collections.singleton("nobody"), Collections.singleton("nobody_group"));
    verifyEntityDomains(applicationId, true);
}

Example 34 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestATSHistoryWithACLs method testDAGACls.

@Test(timeout = 50000)
public void testDAGACls() throws Exception {
    TezClient tezSession = null;
    ApplicationId applicationId;
    String viewAcls = "nobody nobody_group";
    try {
        SleepProcessorConfig spConf = new SleepProcessorConfig(1);
        DAG dag = DAG.create("TezSleepProcessor");
        Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1, Resource.newInstance(256, 1));
        dag.addVertex(vertex);
        DAGAccessControls accessControls = new DAGAccessControls();
        accessControls.setUsersWithViewACLs(Collections.singleton("nobody2"));
        accessControls.setGroupsWithViewACLs(Collections.singleton("nobody_group2"));
        dag.setAccessControls(accessControls);
        TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
        tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
        tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName());
        Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
        remoteFs.mkdirs(remoteStagingDir);
        tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
        tezSession = TezClient.create("TezSleepProcessor", tezConf, true);
        tezSession.start();
        applicationId = tezSession.getAppMasterApplicationId();
        DAGClient dagClient = tezSession.submitDAG(dag);
        DAGStatus dagStatus = dagClient.getDAGStatus(null);
        while (!dagStatus.isCompleted()) {
            LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: " + dagStatus.getState());
            Thread.sleep(500l);
            dagStatus = dagClient.getDAGStatus(null);
        }
        assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
    } finally {
        if (tezSession != null) {
            tezSession.stop();
        }
    }
    TimelineDomain timelineDomain = getDomain(ATSHistoryACLPolicyManager.DOMAIN_ID_PREFIX + applicationId.toString());
    verifyDomainACLs(timelineDomain, Collections.singleton("nobody"), Collections.singleton("nobody_group"));
    timelineDomain = getDomain(ATSHistoryACLPolicyManager.DOMAIN_ID_PREFIX + applicationId.toString() + "_1");
    verifyDomainACLs(timelineDomain, Sets.newHashSet("nobody", "nobody2"), Sets.newHashSet("nobody_group", "nobody_group2"));
    verifyEntityDomains(applicationId, false);
}

Example 35 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestAMRecovery method runDAGAndVerify.

TezCounters runDAGAndVerify(DAG dag, DAGStatus.State finalState) throws Exception {
    tezSession.waitTillReady();
    DAGClient dagClient = tezSession.submitDAG(dag);
    DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(EnumSet.of(StatusGetOpts.GET_COUNTERS));
    Assert.assertEquals(finalState, dagStatus.getState());
    return dagStatus.getDAGCounters();
}