Examples with TezClient org.apache.tez.client.TezClient used on opensource projects

Example 11 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestExternalTezServicesErrors method testFatalError.

private void testFatalError(String methodName, Vertex.VertexExecutionContext lhsExecutionContext, String dagNameSuffix, List<String> expectedDiagMessages) throws IOException, TezException, YarnException, InterruptedException {
    TezConfiguration tezClientConf = new TezConfiguration(extServiceTestHelper.getConfForJobs());
    TezClient tezClient = TezClient.newBuilder(TestExternalTezServicesErrors.class.getSimpleName() + methodName + "_session", tezClientConf).setIsSession(true).setServicePluginDescriptor(servicePluginsDescriptor).build();
    ApplicationId appId = null;
    try {
        tezClient.start();
        LOG.info("TezSessionStarted for " + methodName);
        tezClient.waitTillReady();
        LOG.info("TezSession ready for submission for " + methodName);
        JoinValidateConfigured joinValidate = new JoinValidateConfigured(EXECUTION_CONTEXT_DEFAULT, lhsExecutionContext, EXECUTION_CONTEXT_EXT_SERVICE_PUSH, EXECUTION_CONTEXT_EXT_SERVICE_PUSH, dagNameSuffix);
        DAG dag = joinValidate.createDag(new TezConfiguration(extServiceTestHelper.getConfForJobs()), HASH_JOIN_EXPECTED_RESULT_PATH, HASH_JOIN_OUTPUT_PATH, 3);
        DAGClient dagClient = tezClient.submitDAG(dag);
        DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
        assertEquals(DAGStatus.State.ERROR, dagStatus.getState());
        boolean foundDiag = false;
        for (String diag : dagStatus.getDiagnostics()) {
            foundDiag = checkDiag(diag, expectedDiagMessages);
            if (foundDiag) {
                break;
            }
        }
        appId = tezClient.getAppMasterApplicationId();
        assertTrue(foundDiag);
    } catch (InterruptedException e) {
        e.printStackTrace();
    } finally {
        tezClient.stop();
    }
    // Verify the state of the application.
    if (appId != null) {
        YarnClient yarnClient = YarnClient.createYarnClient();
        try {
            yarnClient.init(tezClientConf);
            yarnClient.start();
            ApplicationReport appReport = yarnClient.getApplicationReport(appId);
            YarnApplicationState appState = appReport.getYarnApplicationState();
            while (!EnumSet.of(YarnApplicationState.FINISHED, YarnApplicationState.FAILED, YarnApplicationState.KILLED).contains(appState)) {
                Thread.sleep(200L);
                appReport = yarnClient.getApplicationReport(appId);
                appState = appReport.getYarnApplicationState();
            }
            // TODO Workaround for YARN-4554. AppReport does not provide diagnostics - need to fetch them from ApplicationAttemptReport
            ApplicationAttemptId appAttemptId = appReport.getCurrentApplicationAttemptId();
            ApplicationAttemptReport appAttemptReport = yarnClient.getApplicationAttemptReport(appAttemptId);
            String diag = appAttemptReport.getDiagnostics();
            assertEquals(FinalApplicationStatus.FAILED, appReport.getFinalApplicationStatus());
            assertEquals(YarnApplicationState.FINISHED, appReport.getYarnApplicationState());
            checkDiag(diag, expectedDiagMessages);
        } finally {
            yarnClient.stop();
        }
    }
}

Example 12 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TezExampleBase method createTezClient.

private TezClient createTezClient(TezConfiguration tezConf) throws IOException, TezException {
    TezClient tezClient = TezClient.create("TezExampleApplication", tezConf);
    if (reconnectAppId != null) {
        ApplicationId appId = TezClient.appIdfromString(reconnectAppId);
        tezClient.getClient(appId);
    } else {
        tezClient.start();
    }
    return tezClient;
}

Example 13 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestLocalMode method testNoSysExitOnSuccessfulDAG.

@Test(timeout = 20000)
public void testNoSysExitOnSuccessfulDAG() throws TezException, InterruptedException, IOException {
    TezConfiguration tezConf1 = new TezConfiguration();
    tezConf1.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
    tezConf1.set("fs.defaultFS", "file:///");
    tezConf1.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
    // Run in non-session mode so that the AM terminates
    TezClient tezClient1 = TezClient.create("commonName", tezConf1, false);
    tezClient1.start();
    DAG dag1 = createSimpleDAG("dag1", SleepProcessor.class.getName());
    DAGClient dagClient1 = tezClient1.submitDAG(dag1);
    dagClient1.waitForCompletion();
    assertEquals(DAGStatus.State.SUCCEEDED, dagClient1.getDAGStatus(null).getState());
    // Sleep for more time than is required for the DAG to complete.
    Thread.sleep((long) (TezConstants.TEZ_DAG_SLEEP_TIME_BEFORE_EXIT * 1.5));
    dagClient1.close();
    tezClient1.stop();
}

Example 14 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestLocalMode method testMultipleClientsWithoutSession.

@Test(timeout = 10000)
public void testMultipleClientsWithoutSession() throws TezException, InterruptedException, IOException {
    TezConfiguration tezConf1 = new TezConfiguration();
    tezConf1.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
    tezConf1.set("fs.defaultFS", "file:///");
    tezConf1.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
    TezClient tezClient1 = TezClient.create("commonName", tezConf1, false);
    tezClient1.start();
    DAG dag1 = createSimpleDAG("dag1", SleepProcessor.class.getName());
    DAGClient dagClient1 = tezClient1.submitDAG(dag1);
    dagClient1.waitForCompletion();
    assertEquals(DAGStatus.State.SUCCEEDED, dagClient1.getDAGStatus(null).getState());
    dagClient1.close();
    tezClient1.stop();
    TezConfiguration tezConf2 = new TezConfiguration();
    tezConf2.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
    tezConf2.set("fs.defaultFS", "file:///");
    tezConf2.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
    DAG dag2 = createSimpleDAG("dag2", SleepProcessor.class.getName());
    TezClient tezClient2 = TezClient.create("commonName", tezConf2, false);
    tezClient2.start();
    DAGClient dagClient2 = tezClient2.submitDAG(dag2);
    dagClient2.waitForCompletion();
    assertEquals(DAGStatus.State.SUCCEEDED, dagClient2.getDAGStatus(null).getState());
    assertFalse(dagClient1.getExecutionContext().equals(dagClient2.getExecutionContext()));
    dagClient2.close();
    tezClient2.stop();
}

Example 15 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestLocalMode method testMultiDAGsOnSession.

@Test(timeout = 30000)
public void testMultiDAGsOnSession() throws IOException, TezException, InterruptedException {
    // two dags will be submitted to session
    int dags = 2;
    String[] inputPaths = new String[dags];
    String[] outputPaths = new String[dags];
    DAGClient[] dagClients = new DAGClient[dags];
    TezConfiguration tezConf = new TezConfiguration();
    tezConf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
    tezConf.set("fs.defaultFS", "file:///");
    tezConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
    TezClient tezClient = TezClient.create("testMultiDAGOnSession", tezConf, true);
    tezClient.start();
    // create inputs and outputs
    FileSystem fs = FileSystem.get(tezConf);
    for (int i = 0; i < dags; i++) {
        inputPaths[i] = new Path(TEST_DIR.getAbsolutePath(), "in-" + i).toString();
        createInputFile(fs, inputPaths[i]);
        outputPaths[i] = new Path(TEST_DIR.getAbsolutePath(), "out-" + i).toString();
    }
    // start testing
    try {
        for (int i = 0; i < inputPaths.length; ++i) {
            DAG dag = OrderedWordCount.createDAG(tezConf, inputPaths[i], outputPaths[i], 1, false, false, // the names of the DAGs must be unique in a session
            ("DAG-Iteration-" + i));
            tezClient.waitTillReady();
            System.out.println("Running dag number " + i);
            dagClients[i] = tezClient.submitDAG(dag);
            // wait to finish
            DAGStatus dagStatus = dagClients[i].waitForCompletion();
            if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
                fail("Iteration " + i + " failed with diagnostics: " + dagStatus.getDiagnostics());
            }
            // verify all dags sharing the same execution context
            if (i > 0) {
                assertTrue(dagClients[i - 1].getExecutionContext().equals(dagClients[i].getExecutionContext()));
            }
        }
    } finally {
        tezClient.stop();
    }
}