Examples with DAGStatus org.apache.tez.dag.api.client.DAGStatus used on opensource projects

Example 36 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestDAGRecovery2 method runDAGAndVerify.

void runDAGAndVerify(DAG dag, DAGStatus.State finalState, TezClient session) throws Exception {
    session.waitTillReady();
    DAGClient dagClient = session.submitDAG(dag);
    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
        LOG.info("Waiting for dag to complete. Sleeping for 500ms." + " DAG name: " + dag.getName() + " DAG appContext: " + dagClient.getExecutionContext() + " Current state: " + dagStatus.getState());
        Thread.sleep(100);
        dagStatus = dagClient.getDAGStatus(null);
    }
    Assert.assertEquals(finalState, dagStatus.getState());
}

Example 37 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestExceptionPropagation method testExceptionPropagationNonSession.

/**
 * verify the diagnostics in {@link DAGStatus} is correct in non-session mode,
 * and also verify that diagnostics from {@link DAGStatus} should match that
 * from {@link ApplicationReport}
 *
 * @throws Exception
 */
@Test(timeout = 120000)
public void testExceptionPropagationNonSession() throws Exception {
    try {
        startMiniTezCluster();
        startNonSessionClient();
        ExceptionLocation exLocation = ExceptionLocation.EM_GetNumSourceTaskPhysicalOutputs;
        LOG.info("NonSession mode, Test for Exception from:" + exLocation.name());
        DAG dag = createDAG(exLocation);
        DAGClient dagClient = tezClient.submitDAG(dag);
        DAGStatus dagStatus = dagClient.waitForCompletion();
        String diagnostics = StringUtils.join(dagStatus.getDiagnostics(), ",");
        LOG.info("Diagnostics:" + diagnostics);
        assertTrue(diagnostics.contains(exLocation.name()));
        // wait for app complete (unregisterApplicationMaster is done)
        ApplicationId appId = tezClient.getAppMasterApplicationId();
        YarnClient yarnClient = YarnClient.createYarnClient();
        yarnClient.init(tezConf);
        yarnClient.start();
        Set<YarnApplicationState> FINAL_APPLICATION_STATES = EnumSet.of(YarnApplicationState.KILLED, YarnApplicationState.FAILED, YarnApplicationState.FINISHED);
        ApplicationReport appReport = null;
        while (true) {
            appReport = yarnClient.getApplicationReport(appId);
            Thread.sleep(1000);
            LOG.info("FinalAppStatus:" + appReport.getFinalApplicationStatus());
            LOG.info("Diagnostics from appReport:" + appReport.getDiagnostics());
            if (FINAL_APPLICATION_STATES.contains(appReport.getYarnApplicationState())) {
                break;
            }
        }
        // wait for 1 second and call getApplicationReport again to ensure get the
        // diagnostics
        // TODO remove it after YARN-2560
        Thread.sleep(1000);
        appReport = yarnClient.getApplicationReport(appId);
        LOG.info("FinalAppStatus:" + appReport.getFinalApplicationStatus());
        LOG.info("Diagnostics from appReport:" + appReport.getDiagnostics());
        assertTrue(appReport.getDiagnostics().contains(exLocation.name()));
        // use "\n" as separator, because we also use it in Tez internally when
        // assembling the application diagnostics.
        assertEquals(StringUtils.join(dagStatus.getDiagnostics(), "\n").trim(), appReport.getDiagnostics().trim());
    } finally {
        stopNonSessionClient();
        Thread.sleep(10 * 1000);
        stopTezMiniCluster();
    }
}

Example 38 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestFaultTolerance method runDAGAndVerify.

void runDAGAndVerify(DAG dag, DAGStatus.State finalState, int checkFailedAttempts, String diagnostics) throws Exception {
    tezSession.waitTillReady();
    DAGClient dagClient = tezSession.submitDAG(dag);
    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {
        LOG.info("Waiting for dag to complete. Sleeping for 500ms." + " DAG name: " + dag.getName() + " DAG appContext: " + dagClient.getExecutionContext() + " Current state: " + dagStatus.getState());
        Thread.sleep(100);
        dagStatus = dagClient.getDAGStatus(null);
    }
    Assert.assertEquals(finalState, dagStatus.getState());
    if (checkFailedAttempts > 0) {
        Assert.assertEquals(checkFailedAttempts, dagStatus.getDAGProgress().getFailedTaskAttemptCount());
    }
    if (diagnostics != null) {
        Assert.assertNotNull(dagStatus.getDiagnostics());
        Assert.assertTrue(Joiner.on(":").join(dagStatus.getDiagnostics()).contains(diagnostics));
    }
}

Example 39 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestTezJobs method testVertexOrder.

@Test(timeout = 60000)
public void testVertexOrder() throws Exception {
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    TezClient tezClient = TezClient.create("TestVertexOrder", tezConf);
    tezClient.start();
    try {
        DAG dag = SimpleTestDAG.createDAGForVertexOrder("dag1", conf);
        DAGClient dagClient = tezClient.submitDAG(dag);
        DAGStatus dagStatus = dagClient.getDAGStatus(null);
        while (!dagStatus.isCompleted()) {
            LOG.info("Waiting for dag to complete. Sleeping for 500ms." + " DAG name: " + dag.getName() + " DAG context: " + dagClient.getExecutionContext() + " Current state: " + dagStatus.getState());
            Thread.sleep(100);
            dagStatus = dagClient.getDAGStatus(null);
        }
        Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState());
        // verify vertex order
        Set<String> resultVertices = dagStatus.getVertexProgress().keySet();
        Assert.assertEquals(6, resultVertices.size());
        int i = 0;
        for (String vertexName : resultVertices) {
            if (i <= 1) {
                Assert.assertTrue(vertexName.equals("v1") || vertexName.equals("v2"));
            } else if (i == 2) {
                Assert.assertTrue(vertexName.equals("v3"));
            } else if (i <= 4) {
                Assert.assertTrue(vertexName.equals("v4") || vertexName.equals("v5"));
            } else {
                Assert.assertTrue(vertexName.equals("v6"));
            }
            i++;
        }
    } finally {
        if (tezClient != null) {
            tezClient.stop();
        }
    }
}

Example 40 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class FilterLinesByWord method run.

@Override
public int run(String[] args) throws Exception {
    Configuration conf = getConf();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    Credentials credentials = new Credentials();
    boolean generateSplitsInClient = false;
    SplitsInClientOptionParser splitCmdLineParser = new SplitsInClientOptionParser();
    try {
        generateSplitsInClient = splitCmdLineParser.parse(otherArgs, false);
        otherArgs = splitCmdLineParser.getRemainingArgs();
    } catch (ParseException e1) {
        System.err.println("Invalid options");
        printUsage();
        return 2;
    }
    if (otherArgs.length != 3) {
        printUsage();
        return 2;
    }
    String inputPath = otherArgs[0];
    String outputPath = otherArgs[1];
    String filterWord = otherArgs[2];
    FileSystem fs = FileSystem.get(conf);
    if (fs.exists(new Path(outputPath))) {
        System.err.println("Output directory : " + outputPath + " already exists");
        return 2;
    }
    TezConfiguration tezConf = new TezConfiguration(conf);
    fs.getWorkingDirectory();
    Path stagingDir = new Path(fs.getWorkingDirectory(), UUID.randomUUID().toString());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());
    TezClientUtils.ensureStagingDirExists(tezConf, stagingDir);
    String jarPath = ClassUtil.findContainingJar(FilterLinesByWord.class);
    if (jarPath == null) {
        throw new TezUncheckedException("Could not find any jar containing" + FilterLinesByWord.class.getName() + " in the classpath");
    }
    Path remoteJarPath = fs.makeQualified(new Path(stagingDir, "dag_job.jar"));
    fs.copyFromLocalFile(new Path(jarPath), remoteJarPath);
    FileStatus remoteJarStatus = fs.getFileStatus(remoteJarPath);
    TokenCache.obtainTokensForNamenodes(credentials, new Path[] { remoteJarPath }, conf);
    Map<String, LocalResource> commonLocalResources = new TreeMap<String, LocalResource>();
    LocalResource dagJarLocalRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(remoteJarPath), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, remoteJarStatus.getLen(), remoteJarStatus.getModificationTime());
    commonLocalResources.put("dag_job.jar", dagJarLocalRsrc);
    TezClient tezSession = TezClient.create("FilterLinesByWordSession", tezConf, commonLocalResources, credentials);
    // Why do I need to start the TezSession.
    tezSession.start();
    Configuration stage1Conf = new JobConf(conf);
    stage1Conf.set(FILTER_PARAM_NAME, filterWord);
    Configuration stage2Conf = new JobConf(conf);
    stage2Conf.set(FileOutputFormat.OUTDIR, outputPath);
    stage2Conf.setBoolean("mapred.mapper.new-api", false);
    UserPayload stage1Payload = TezUtils.createUserPayloadFromConf(stage1Conf);
    // Setup stage1 Vertex
    Vertex stage1Vertex = Vertex.create("stage1", ProcessorDescriptor.create(FilterByWordInputProcessor.class.getName()).setUserPayload(stage1Payload)).addTaskLocalFiles(commonLocalResources);
    DataSourceDescriptor dsd;
    if (generateSplitsInClient) {
        // TODO TEZ-1406. Dont' use MRInputLegacy
        stage1Conf.set(FileInputFormat.INPUT_DIR, inputPath);
        stage1Conf.setBoolean("mapred.mapper.new-api", false);
        dsd = MRInputHelpers.configureMRInputWithLegacySplitGeneration(stage1Conf, stagingDir, true);
    } else {
        dsd = MRInputLegacy.createConfigBuilder(stage1Conf, TextInputFormat.class, inputPath).groupSplits(false).build();
    }
    stage1Vertex.addDataSource("MRInput", dsd);
    // Setup stage2 Vertex
    Vertex stage2Vertex = Vertex.create("stage2", ProcessorDescriptor.create(FilterByWordOutputProcessor.class.getName()).setUserPayload(TezUtils.createUserPayloadFromConf(stage2Conf)), 1);
    stage2Vertex.addTaskLocalFiles(commonLocalResources);
    // Configure the Output for stage2
    OutputDescriptor od = OutputDescriptor.create(MROutput.class.getName()).setUserPayload(TezUtils.createUserPayloadFromConf(stage2Conf));
    OutputCommitterDescriptor ocd = OutputCommitterDescriptor.create(MROutputCommitter.class.getName());
    stage2Vertex.addDataSink("MROutput", DataSinkDescriptor.create(od, ocd, null));
    UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig.newBuilder(Text.class.getName(), TextLongPair.class.getName()).setFromConfiguration(tezConf).build();
    DAG dag = DAG.create("FilterLinesByWord");
    Edge edge = Edge.create(stage1Vertex, stage2Vertex, edgeConf.createDefaultBroadcastEdgeProperty());
    dag.addVertex(stage1Vertex).addVertex(stage2Vertex).addEdge(edge);
    LOG.info("Submitting DAG to Tez Session");
    DAGClient dagClient = tezSession.submitDAG(dag);
    LOG.info("Submitted DAG to Tez Session");
    DAGStatus dagStatus = null;
    String[] vNames = { "stage1", "stage2" };
    try {
        while (true) {
            dagStatus = dagClient.getDAGStatus(null);
            if (dagStatus.getState() == DAGStatus.State.RUNNING || dagStatus.getState() == DAGStatus.State.SUCCEEDED || dagStatus.getState() == DAGStatus.State.FAILED || dagStatus.getState() == DAGStatus.State.KILLED || dagStatus.getState() == DAGStatus.State.ERROR) {
                break;
            }
            try {
                Thread.sleep(500);
            } catch (InterruptedException e) {
            // continue;
            }
        }
        while (dagStatus.getState() == DAGStatus.State.RUNNING) {
            try {
                ExampleDriver.printDAGStatus(dagClient, vNames);
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException e) {
                // continue;
                }
                dagStatus = dagClient.getDAGStatus(null);
            } catch (TezException e) {
                LOG.error("Failed to get application progress. Exiting");
                return -1;
            }
        }
        dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
    } finally {
        fs.delete(stagingDir, true);
        tezSession.stop();
    }
    ExampleDriver.printDAGStatus(dagClient, vNames, true, true);
    LOG.info("Application completed. " + "FinalState=" + dagStatus.getState());
    return dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1;
}