Examples with JoinValidateConfigured org.apache.tez.examples.JoinValidateConfigured used on opensource projects

Example 1 with JoinValidateConfigured

use of org.apache.tez.examples.JoinValidateConfigured in project tez by apache.

the class TestExternalTezServicesErrors method testFatalError.

private void testFatalError(String methodName, Vertex.VertexExecutionContext lhsExecutionContext, String dagNameSuffix, List<String> expectedDiagMessages) throws IOException, TezException, YarnException, InterruptedException {
    TezConfiguration tezClientConf = new TezConfiguration(extServiceTestHelper.getConfForJobs());
    TezClient tezClient = TezClient.newBuilder(TestExternalTezServicesErrors.class.getSimpleName() + methodName + "_session", tezClientConf).setIsSession(true).setServicePluginDescriptor(servicePluginsDescriptor).build();
    ApplicationId appId = null;
    try {
        tezClient.start();
        LOG.info("TezSessionStarted for " + methodName);
        tezClient.waitTillReady();
        LOG.info("TezSession ready for submission for " + methodName);
        JoinValidateConfigured joinValidate = new JoinValidateConfigured(EXECUTION_CONTEXT_DEFAULT, lhsExecutionContext, EXECUTION_CONTEXT_EXT_SERVICE_PUSH, EXECUTION_CONTEXT_EXT_SERVICE_PUSH, dagNameSuffix);
        DAG dag = joinValidate.createDag(new TezConfiguration(extServiceTestHelper.getConfForJobs()), HASH_JOIN_EXPECTED_RESULT_PATH, HASH_JOIN_OUTPUT_PATH, 3);
        DAGClient dagClient = tezClient.submitDAG(dag);
        DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
        assertEquals(DAGStatus.State.ERROR, dagStatus.getState());
        boolean foundDiag = false;
        for (String diag : dagStatus.getDiagnostics()) {
            foundDiag = checkDiag(diag, expectedDiagMessages);
            if (foundDiag) {
                break;
            }
        }
        appId = tezClient.getAppMasterApplicationId();
        assertTrue(foundDiag);
    } catch (InterruptedException e) {
        e.printStackTrace();
    } finally {
        tezClient.stop();
    }
    // Verify the state of the application.
    if (appId != null) {
        YarnClient yarnClient = YarnClient.createYarnClient();
        try {
            yarnClient.init(tezClientConf);
            yarnClient.start();
            ApplicationReport appReport = yarnClient.getApplicationReport(appId);
            YarnApplicationState appState = appReport.getYarnApplicationState();
            while (!EnumSet.of(YarnApplicationState.FINISHED, YarnApplicationState.FAILED, YarnApplicationState.KILLED).contains(appState)) {
                Thread.sleep(200L);
                appReport = yarnClient.getApplicationReport(appId);
                appState = appReport.getYarnApplicationState();
            }
            // TODO Workaround for YARN-4554. AppReport does not provide diagnostics - need to fetch them from ApplicationAttemptReport
            ApplicationAttemptId appAttemptId = appReport.getCurrentApplicationAttemptId();
            ApplicationAttemptReport appAttemptReport = yarnClient.getApplicationAttemptReport(appAttemptId);
            String diag = appAttemptReport.getDiagnostics();
            assertEquals(FinalApplicationStatus.FAILED, appReport.getFinalApplicationStatus());
            assertEquals(YarnApplicationState.FINISHED, appReport.getYarnApplicationState());
            checkDiag(diag, expectedDiagMessages);
        } finally {
            yarnClient.stop();
        }
    }
}

Example 2 with JoinValidateConfigured

use of org.apache.tez.examples.JoinValidateConfigured in project tez by apache.

the class TestExternalTezServicesErrors method runAndVerifyForNonFatalErrors.

private void runAndVerifyForNonFatalErrors(TezClient tezClient, String componentName, Vertex.VertexExecutionContext lhsContext) throws TezException, InterruptedException, IOException {
    LOG.info("Running JoinValidate with componentName reportNonFatalException");
    JoinValidateConfigured joinValidate = new JoinValidateConfigured(EXECUTION_CONTEXT_DEFAULT, lhsContext, EXECUTION_CONTEXT_EXT_SERVICE_PUSH, EXECUTION_CONTEXT_EXT_SERVICE_PUSH, componentName);
    DAG dag = joinValidate.createDag(new TezConfiguration(extServiceTestHelper.getConfForJobs()), HASH_JOIN_EXPECTED_RESULT_PATH, HASH_JOIN_OUTPUT_PATH, 3);
    DAGClient dagClient = tezClient.submitDAG(dag);
    DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
    assertEquals(DAGStatus.State.FAILED, dagStatus.getState());
    boolean foundDiag = false;
    for (String diag : dagStatus.getDiagnostics()) {
        if (diag.contains(ErrorPluginConfiguration.REPORT_NONFATAL_ERROR_MESSAGE) && diag.contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name())) {
            foundDiag = true;
            break;
        }
    }
    assertTrue(foundDiag);
}

Example 3 with JoinValidateConfigured

use of org.apache.tez.examples.JoinValidateConfigured in project tez by apache.

the class TestExternalTezServices method runJoinValidate.

private void runJoinValidate(String name, int extExpectedCount, VertexExecutionContext lhsContext, VertexExecutionContext rhsContext, VertexExecutionContext validateContext) throws Exception {
    int externalSubmissionCount = extServiceTestHelper.getTezTestServiceCluster().getNumSubmissions();
    TezConfiguration tezConf = new TezConfiguration(extServiceTestHelper.getConfForJobs());
    JoinValidateConfigured joinValidate = new JoinValidateConfigured(EXECUTION_CONTEXT_DEFAULT, lhsContext, rhsContext, validateContext, name);
    String[] validateArgs = new String[] { "-disableSplitGrouping", HASH_JOIN_EXPECTED_RESULT_PATH.toString(), HASH_JOIN_OUTPUT_PATH.toString(), "3" };
    assertEquals(0, joinValidate.run(tezConf, validateArgs, extServiceTestHelper.getSharedTezClient()));
    // Ensure this was actually submitted to the external cluster
    assertEquals(extExpectedCount, (extServiceTestHelper.getTezTestServiceCluster().getNumSubmissions() - externalSubmissionCount));
}

Example 4 with JoinValidateConfigured

use of org.apache.tez.examples.JoinValidateConfigured in project tez by apache.

the class TestExtServicesWithLocalMode method runJoinValidate.

private void runJoinValidate(TezClient tezClient, String name, int extExpectedCount, Vertex.VertexExecutionContext lhsContext, Vertex.VertexExecutionContext rhsContext, Vertex.VertexExecutionContext validateContext) throws Exception {
    int externalSubmissionCount = tezTestServiceCluster.getNumSubmissions();
    TezConfiguration tezConf = new TezConfiguration(confForJobs);
    JoinValidateConfigured joinValidate = new JoinValidateConfigured(null, lhsContext, rhsContext, validateContext, name);
    String[] validateArgs = new String[] { "-disableSplitGrouping", HASH_JOIN_EXPECTED_RESULT_PATH.toString(), HASH_JOIN_OUTPUT_PATH.toString(), "3" };
    assertEquals(0, joinValidate.run(tezConf, validateArgs, tezClient));
    // Ensure this was actually submitted to the external cluster
    assertEquals(extExpectedCount, (tezTestServiceCluster.getNumSubmissions() - externalSubmissionCount));
}