Examples with DAGStatus org.apache.tez.dag.api.client.DAGStatus used on opensource projects

Example 11 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestMockDAGAppMaster method testBasicStatisticsMemory.

@Ignore
@Test(timeout = 60000)
public void testBasicStatisticsMemory() throws Exception {
    Logger.getRootLogger().setLevel(Level.WARN);
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false);
    tezClient.start();
    final String vAName = "abcdefghijklmnopqrstuvwxyz";
    int numTasks = 10000;
    int numSources = 10;
    IOStatistics ioStats = new IOStatistics();
    ioStats.setDataSize(1);
    ioStats.setItemsProcessed(1);
    TaskStatistics vAStats = new TaskStatistics();
    DAG dag = DAG.create("testBasicStatisticsMemory");
    Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), numTasks);
    for (int i = 0; i < numSources; ++i) {
        final String sourceName = i + vAName;
        vA.addDataSource(sourceName, DataSourceDescriptor.create(InputDescriptor.create(sourceName), null, null));
        vAStats.addIO(sourceName, ioStats);
    }
    dag.addVertex(vA);
    ByteArrayOutputStream bosA = new ByteArrayOutputStream();
    DataOutput outA = new DataOutputStream(bosA);
    vAStats.write(outA);
    final byte[] payloadA = bosA.toByteArray();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    mockApp.statsDelegate = new StatisticsDelegate() {

        @Override
        public TaskStatistics getStatistics(TaskSpec taskSpec) {
            byte[] payload = payloadA;
            TaskStatistics stats = new TaskStatistics();
            final DataInputByteBuffer in = new DataInputByteBuffer();
            in.reset(ByteBuffer.wrap(payload));
            try {
                // this ensures that the serde code path is covered.
                stats.readFields(in);
            } catch (IOException e) {
                Assert.fail(e.getMessage());
            }
            return stats;
        }
    };
    mockApp.doSleep = false;
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockLauncher.startScheduling(true);
    DAGStatus status = dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
    Assert.assertEquals(numTasks, dagImpl.getVertex(vAName).getStatistics().getInputStatistics(0 + vAName).getDataSize());
    Assert.assertEquals(numTasks, dagImpl.getVertex(vAName).getStatistics().getInputStatistics(0 + vAName).getItemsProcessed());
    checkMemory(dag.getName(), mockApp);
    tezClient.stop();
}

Example 12 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestExternalTezServicesErrors method testFatalError.

private void testFatalError(String methodName, Vertex.VertexExecutionContext lhsExecutionContext, String dagNameSuffix, List<String> expectedDiagMessages) throws IOException, TezException, YarnException, InterruptedException {
    TezConfiguration tezClientConf = new TezConfiguration(extServiceTestHelper.getConfForJobs());
    TezClient tezClient = TezClient.newBuilder(TestExternalTezServicesErrors.class.getSimpleName() + methodName + "_session", tezClientConf).setIsSession(true).setServicePluginDescriptor(servicePluginsDescriptor).build();
    ApplicationId appId = null;
    try {
        tezClient.start();
        LOG.info("TezSessionStarted for " + methodName);
        tezClient.waitTillReady();
        LOG.info("TezSession ready for submission for " + methodName);
        JoinValidateConfigured joinValidate = new JoinValidateConfigured(EXECUTION_CONTEXT_DEFAULT, lhsExecutionContext, EXECUTION_CONTEXT_EXT_SERVICE_PUSH, EXECUTION_CONTEXT_EXT_SERVICE_PUSH, dagNameSuffix);
        DAG dag = joinValidate.createDag(new TezConfiguration(extServiceTestHelper.getConfForJobs()), HASH_JOIN_EXPECTED_RESULT_PATH, HASH_JOIN_OUTPUT_PATH, 3);
        DAGClient dagClient = tezClient.submitDAG(dag);
        DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
        assertEquals(DAGStatus.State.ERROR, dagStatus.getState());
        boolean foundDiag = false;
        for (String diag : dagStatus.getDiagnostics()) {
            foundDiag = checkDiag(diag, expectedDiagMessages);
            if (foundDiag) {
                break;
            }
        }
        appId = tezClient.getAppMasterApplicationId();
        assertTrue(foundDiag);
    } catch (InterruptedException e) {
        e.printStackTrace();
    } finally {
        tezClient.stop();
    }
    // Verify the state of the application.
    if (appId != null) {
        YarnClient yarnClient = YarnClient.createYarnClient();
        try {
            yarnClient.init(tezClientConf);
            yarnClient.start();
            ApplicationReport appReport = yarnClient.getApplicationReport(appId);
            YarnApplicationState appState = appReport.getYarnApplicationState();
            while (!EnumSet.of(YarnApplicationState.FINISHED, YarnApplicationState.FAILED, YarnApplicationState.KILLED).contains(appState)) {
                Thread.sleep(200L);
                appReport = yarnClient.getApplicationReport(appId);
                appState = appReport.getYarnApplicationState();
            }
            // TODO Workaround for YARN-4554. AppReport does not provide diagnostics - need to fetch them from ApplicationAttemptReport
            ApplicationAttemptId appAttemptId = appReport.getCurrentApplicationAttemptId();
            ApplicationAttemptReport appAttemptReport = yarnClient.getApplicationAttemptReport(appAttemptId);
            String diag = appAttemptReport.getDiagnostics();
            assertEquals(FinalApplicationStatus.FAILED, appReport.getFinalApplicationStatus());
            assertEquals(YarnApplicationState.FINISHED, appReport.getYarnApplicationState());
            checkDiag(diag, expectedDiagMessages);
        } finally {
            yarnClient.stop();
        }
    }
}

Example 13 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TestExternalTezServicesErrors method runAndVerifyForNonFatalErrors.

private void runAndVerifyForNonFatalErrors(TezClient tezClient, String componentName, Vertex.VertexExecutionContext lhsContext) throws TezException, InterruptedException, IOException {
    LOG.info("Running JoinValidate with componentName reportNonFatalException");
    JoinValidateConfigured joinValidate = new JoinValidateConfigured(EXECUTION_CONTEXT_DEFAULT, lhsContext, EXECUTION_CONTEXT_EXT_SERVICE_PUSH, EXECUTION_CONTEXT_EXT_SERVICE_PUSH, componentName);
    DAG dag = joinValidate.createDag(new TezConfiguration(extServiceTestHelper.getConfForJobs()), HASH_JOIN_EXPECTED_RESULT_PATH, HASH_JOIN_OUTPUT_PATH, 3);
    DAGClient dagClient = tezClient.submitDAG(dag);
    DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
    assertEquals(DAGStatus.State.FAILED, dagStatus.getState());
    boolean foundDiag = false;
    for (String diag : dagStatus.getDiagnostics()) {
        if (diag.contains(ErrorPluginConfiguration.REPORT_NONFATAL_ERROR_MESSAGE) && diag.contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name())) {
            foundDiag = true;
            break;
        }
    }
    assertTrue(foundDiag);
}

Example 14 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class JoinValidate method runJob.

@Override
protected int runJob(String[] args, TezConfiguration tezConf, TezClient tezClient) throws Exception {
    LOG.info("Running JoinValidate");
    String lhsDir = args[0];
    String rhsDir = args[1];
    int numPartitions = 1;
    if (args.length == 3) {
        numPartitions = Integer.parseInt(args[2]);
    }
    if (numPartitions <= 0) {
        System.err.println("NumPartitions must be > 0");
        return 4;
    }
    Path lhsPath = new Path(lhsDir);
    Path rhsPath = new Path(rhsDir);
    DAG dag = createDag(tezConf, lhsPath, rhsPath, numPartitions);
    tezClient.waitTillReady();
    DAGClient dagClient = tezClient.submitDAG(dag);
    Set<StatusGetOpts> getOpts = Sets.newHashSet();
    if (isCountersLog()) {
        getOpts.add(StatusGetOpts.GET_COUNTERS);
    }
    DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(getOpts);
    if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
        LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics());
        return -1;
    } else {
        dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
        TezCounter counter = dagStatus.getDAGCounters().findCounter(COUNTER_GROUP_NAME, MISSING_KEY_COUNTER_NAME);
        if (counter == null) {
            LOG.info("Unable to determing equality");
            return -2;
        } else {
            if (counter.getValue() != 0) {
                LOG.info("Validate failed. The two sides are not equivalent");
                return -3;
            } else {
                LOG.info("Validation successful. The two sides are equivalent");
                return 0;
            }
        }
    }
}

Example 15 with DAGStatus

use of org.apache.tez.dag.api.client.DAGStatus in project tez by apache.

the class TezExampleBase method runDag.

/**
 * @param dag           the dag to execute
 * @param printCounters whether to print counters or not
 * @param logger        the logger to use while printing diagnostics
 * @return Zero indicates success, non-zero indicates failure
 * @throws TezException
 * @throws InterruptedException
 * @throws IOException
 */
public int runDag(DAG dag, boolean printCounters, Logger logger) throws TezException, InterruptedException, IOException {
    tezClientInternal.waitTillReady();
    CallerContext callerContext = CallerContext.create("TezExamples", "Tez Example DAG: " + dag.getName());
    ApplicationId appId = tezClientInternal.getAppMasterApplicationId();
    if (hadoopShim == null) {
        Configuration conf = (getConf() == null ? new Configuration(false) : getConf());
        hadoopShim = new HadoopShimsLoader(conf).getHadoopShim();
    }
    if (appId != null) {
        TezUtilsInternal.setHadoopCallerContext(hadoopShim, appId);
        callerContext.setCallerIdAndType(appId.toString(), "TezExampleApplication");
    }
    dag.setCallerContext(callerContext);
    DAGClient dagClient = tezClientInternal.submitDAG(dag);
    Set<StatusGetOpts> getOpts = Sets.newHashSet();
    if (printCounters) {
        getOpts.add(StatusGetOpts.GET_COUNTERS);
    }
    DAGStatus dagStatus;
    dagStatus = dagClient.waitForCompletionWithStatusUpdates(getOpts);
    if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
        logger.info("DAG diagnostics: " + dagStatus.getDiagnostics());
        return -1;
    }
    return 0;
}