Examples with TezClient org.apache.tez.client.TezClient used on opensource projects

Example 51 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestTaskErrorsUsingLocalMode method testFatalErrorReported.

@Test(timeout = 20000)
public void testFatalErrorReported() throws IOException, TezException, InterruptedException {
    TezClient tezClient = getTezClient("testFatalErrorReported");
    DAGClient dagClient = null;
    try {
        FailingProcessor.configureForFatalFail();
        DAG dag = DAG.create("testFatalErrorReportedDag").addVertex(Vertex.create(VERTEX_NAME, ProcessorDescriptor.create(FailingProcessor.class.getName()), 1));
        dagClient = tezClient.submitDAG(dag);
        dagClient.waitForCompletion();
        assertEquals(DAGStatus.State.FAILED, dagClient.getDAGStatus(null).getState());
        assertEquals(1, dagClient.getVertexStatus(VERTEX_NAME, null).getProgress().getFailedTaskAttemptCount());
    } finally {
        if (dagClient != null) {
            dagClient.close();
        }
        tezClient.stop();
    }
}

Example 52 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestHistoryParser method runWordCount.

private String runWordCount(String tokenizerProcessor, String summationProcessor, String dagName, boolean withTimeline) throws Exception {
    // HDFS path
    Path outputLoc = new Path("/tmp/outPath_" + System.currentTimeMillis());
    DataSourceDescriptor dataSource = MRInput.createConfigBuilder(conf, TextInputFormat.class, inputLoc.toString()).build();
    DataSinkDescriptor dataSink = MROutput.createConfigBuilder(conf, TextOutputFormat.class, outputLoc.toString()).build();
    Vertex tokenizerVertex = Vertex.create(TOKENIZER, ProcessorDescriptor.create(tokenizerProcessor)).addDataSource(INPUT, dataSource);
    OrderedPartitionedKVEdgeConfig edgeConf = OrderedPartitionedKVEdgeConfig.newBuilder(Text.class.getName(), IntWritable.class.getName(), HashPartitioner.class.getName()).build();
    Vertex summationVertex = Vertex.create(SUMMATION, ProcessorDescriptor.create(summationProcessor), 1).addDataSink(OUTPUT, dataSink);
    // Create DAG and add the vertices. Connect the producer and consumer vertices via the edge
    DAG dag = DAG.create(dagName);
    dag.addVertex(tokenizerVertex).addVertex(summationVertex).addEdge(Edge.create(tokenizerVertex, summationVertex, edgeConf.createDefaultEdgeProperty()));
    TezClient tezClient = getTezClient(withTimeline);
    // Update Caller Context
    CallerContext callerContext = CallerContext.create("TezExamples", "Tez WordCount Example Job");
    ApplicationId appId = tezClient.getAppMasterApplicationId();
    if (appId == null) {
        appId = ApplicationId.newInstance(1001l, 1);
    }
    callerContext.setCallerIdAndType(appId.toString(), "TezApplication");
    dag.setCallerContext(callerContext);
    DAGClient client = tezClient.submitDAG(dag);
    client.waitForCompletionWithStatusUpdates(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
    TezDAGID tezDAGID = TezDAGID.getInstance(tezClient.getAppMasterApplicationId(), 1);
    if (tezClient != null) {
        tezClient.stop();
    }
    return tezDAGID.toString();
}

Example 53 with TezClient

use of org.apache.tez.client.TezClient in project tez by apache.

the class TestHistoryParser method getTezClient.

TezClient getTezClient(boolean withTimeline) throws Exception {
    TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig());
    if (withTimeline) {
        tezConf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, withTimeline);
        tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName());
    } else {
        tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, SimpleHistoryLoggingService.class.getName());
    }
    tezConf.setBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS, true);
    TezClient tezClient = TezClient.create("WordCount", tezConf, false);
    tezClient.start();
    tezClient.waitTillReady();
    return tezClient;
}

Example 54 with TezClient

use of org.apache.tez.client.TezClient in project hive by apache.

the class TestTezTask method testSubmitOnPoolSession.

@Test
public void testSubmitOnPoolSession() throws Exception {
    DAG dag = DAG.create("test");
    // Move session to TezSessionPool, reopen will handle it
    SampleTezSessionState tezSessionPoolSession = mock(SampleTezSessionState.class);
    TezClient tezClient = mock(TezClient.class);
    when(tezSessionPoolSession.reopen()).thenThrow(new HiveException("Dag cannot be submitted"));
    doNothing().when(tezSessionPoolSession).returnToSessionManager();
    when(tezSessionPoolSession.getSession()).thenReturn(tezClient);
    when(tezSessionPoolSession.isDefault()).thenReturn(true);
    when(tezClient.submitDAG(any(DAG.class))).thenThrow(new SessionNotRunning(""));
    boolean isException = false;
    try {
        task.submit(dag, Ref.from(tezSessionPoolSession));
    } catch (Exception e) {
        isException = true;
        verify(tezClient, times(1)).submitDAG(any(DAG.class));
        verify(tezSessionPoolSession, times(2)).reopen();
        verify(tezSessionPoolSession, times(0)).destroy();
        verify(tezSessionPoolSession, times(1)).returnToSessionManager();
    }
    assertTrue(isException);
}

Example 55 with TezClient

use of org.apache.tez.client.TezClient in project hive by apache.

the class TestTezTask method testSubmitOnNonPoolSession.

@Test
public void testSubmitOnNonPoolSession() throws Exception {
    DAG dag = DAG.create("test");
    // Destroy session incase of non-pool tez session
    TezSessionState tezSessionState = mock(TezSessionState.class);
    TezClient tezClient = mock(TezClient.class);
    when(tezSessionState.reopen()).thenThrow(new HiveException("Dag cannot be submitted"));
    when(tezSessionState.getSession()).thenReturn(tezClient);
    when(tezClient.submitDAG(any(DAG.class))).thenThrow(new SessionNotRunning(""));
    doNothing().when(tezSessionState).destroy();
    boolean isException = false;
    try {
        task.submit(dag, Ref.from(tezSessionState));
    } catch (Exception e) {
        isException = true;
        verify(tezClient, times(1)).submitDAG(any(DAG.class));
        verify(tezSessionState, times(2)).reopen();
        verify(tezSessionState, times(1)).destroy();
        verify(tezSessionState, times(0)).returnToSessionManager();
    }
    assertTrue(isException);
}