Examples with DAG org.apache.tez.dag.app.dag.DAG used on opensource projects

Example 16 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class TestHistoryEventHandler method createHandler.

private HistoryEventHandler createHandler(HistoryLogLevel logLevel) {
    Configuration conf = new Configuration(baseConfig);
    conf.setBoolean(TezConfiguration.DAG_RECOVERY_ENABLED, false);
    conf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, InMemoryHistoryLoggingService.class.getName());
    if (logLevel != null) {
        conf.setEnum(TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL, logLevel);
    }
    DAG dag = mock(DAG.class);
    when(dag.getConf()).thenReturn(conf);
    AppContext appContext = mock(AppContext.class);
    when(appContext.getApplicationID()).thenReturn(appId);
    when(appContext.getHadoopShim()).thenReturn(new HadoopShim() {
    });
    when(appContext.getAMConf()).thenReturn(conf);
    when(appContext.getCurrentDAG()).thenReturn(dag);
    HistoryEventHandler handler = new HistoryEventHandler(appContext);
    handler.init(conf);
    return handler;
}

Example 17 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class Utils method processNonFatalServiceErrorReport.

public static void processNonFatalServiceErrorReport(String entityString, ServicePluginError servicePluginError, String diagnostics, DagInfo dagInfo, AppContext appContext, String componentName) {
    String message = "Error reported by " + componentName + " [" + entityString + "][" + servicePluginError + "] " + (diagnostics == null ? "" : diagnostics);
    if (dagInfo != null) {
        DAG dag = appContext.getCurrentDAG();
        if (dag != null && dag.getID().getId() == dagInfo.getIndex()) {
            TezDAGID dagId = dag.getID();
            // Send a kill message only if it is the same dag.
            LOG.warn(message + ", Failing dag: [" + dagInfo.getName() + ", " + dagId + "]");
            sendEvent(appContext, new DAGEventTerminateDag(dagId, DAGTerminationCause.SERVICE_PLUGIN_ERROR, message));
        }
    } else {
        LOG.warn("No current dag name provided. Not acting on " + message);
    }
}

Example 18 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class DAGClientHandler method tryKillDAG.

public void tryKillDAG(String dagIdStr) throws TezException {
    DAG dag = getDAG(dagIdStr);
    String message = "Sending client kill from " + getClientInfo() + " to dag " + dagIdStr;
    LOG.info(message);
    dagAppMaster.tryKillDAG(dag, message);
}

Example 19 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class TaskCommunicatorContextImpl method getFirstAttemptStartTime.

@Override
public long getFirstAttemptStartTime(String vertexName, int taskIndex) {
    Preconditions.checkArgument(vertexName != null, "VertexName must be specified");
    Preconditions.checkArgument(taskIndex >= 0, "TaskIndex must be > 0");
    DAG dag = getDag();
    Vertex vertex = dag.getVertex(vertexName);
    Task task = vertex.getTask(taskIndex);
    return task.getFirstAttemptStartTime();
}

Example 20 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class TestDAGClientHandler method testDAGClientHandler.

@Test(timeout = 5000)
public void testDAGClientHandler() throws TezException {
    TezDAGID mockTezDAGId = mock(TezDAGID.class);
    when(mockTezDAGId.getId()).thenReturn(1);
    when(mockTezDAGId.toString()).thenReturn("dag_9999_0001_1");
    DAG mockDAG = mock(DAG.class);
    when(mockDAG.getID()).thenReturn(mockTezDAGId);
    DAGStatusBuilder mockDagStatusBuilder = mock(DAGStatusBuilder.class);
    when(mockDAG.getDAGStatus(anySetOf(StatusGetOpts.class))).thenReturn(mockDagStatusBuilder);
    VertexStatusBuilder mockVertexStatusBuilder = mock(VertexStatusBuilder.class);
    when(mockDAG.getVertexStatus(anyString(), anySetOf(StatusGetOpts.class))).thenReturn(mockVertexStatusBuilder);
    DAGAppMaster mockDagAM = mock(DAGAppMaster.class);
    when(mockDagAM.getState()).thenReturn(DAGAppMasterState.RUNNING);
    AppContext mockAppContext = mock(AppContext.class);
    when(mockDagAM.getContext()).thenReturn(mockAppContext);
    when(mockDagAM.getContext().getCurrentDAG()).thenReturn(mockDAG);
    when(mockAppContext.getClock()).thenReturn(new SystemClock());
    DAGClientHandler dagClientHandler = new DAGClientHandler(mockDagAM);
    // getAllDAGs()
    assertEquals(1, dagClientHandler.getAllDAGs().size());
    assertEquals("dag_9999_0001_1", dagClientHandler.getAllDAGs().get(0));
    // getDAGStatus
    try {
        dagClientHandler.getDAGStatus("dag_9999_0001_2", Sets.newSet(StatusGetOpts.GET_COUNTERS));
        fail("should not come here");
    } catch (TezException e) {
        assertTrue(e.getMessage().contains("Unknown dagId"));
    }
    DAGStatus dagStatus = dagClientHandler.getDAGStatus("dag_9999_0001_1", Sets.newSet(StatusGetOpts.GET_COUNTERS));
    assertEquals(mockDagStatusBuilder, dagStatus);
    // getVertexStatus
    try {
        dagClientHandler.getVertexStatus("dag_9999_0001_2", "v1", Sets.newSet(StatusGetOpts.GET_COUNTERS));
        fail("should not come here");
    } catch (TezException e) {
        assertTrue(e.getMessage().contains("Unknown dagId"));
    }
    VertexStatus vertexStatus = dagClientHandler.getVertexStatus("dag_9999_0001_1", "v1", Sets.newSet(StatusGetOpts.GET_COUNTERS));
    assertEquals(mockVertexStatusBuilder, vertexStatus);
    // getTezAppMasterStatus
    when(mockDagAM.isSession()).thenReturn(false);
    assertEquals(TezAppMasterStatus.RUNNING, dagClientHandler.getTezAppMasterStatus());
    when(mockDagAM.isSession()).thenReturn(true);
    when(mockDagAM.getState()).thenReturn(DAGAppMasterState.INITED);
    assertEquals(TezAppMasterStatus.INITIALIZING, dagClientHandler.getTezAppMasterStatus());
    when(mockDagAM.getState()).thenReturn(DAGAppMasterState.ERROR);
    assertEquals(TezAppMasterStatus.SHUTDOWN, dagClientHandler.getTezAppMasterStatus());
    // tryKillDAG
    try {
        dagClientHandler.tryKillDAG("dag_9999_0001_2");
        fail("should not come here");
    } catch (TezException e) {
        assertTrue(e.getMessage().contains("Unknown dagId"));
    }
    dagClientHandler.tryKillDAG("dag_9999_0001_1");
    ArgumentCaptor<DAG> eventCaptor = ArgumentCaptor.forClass(DAG.class);
    verify(mockDagAM, times(1)).tryKillDAG(eventCaptor.capture(), contains("Sending client kill from"));
    assertEquals(1, eventCaptor.getAllValues().size());
    assertTrue(eventCaptor.getAllValues().get(0) instanceof DAG);
    assertEquals("dag_9999_0001_1", ((DAG) eventCaptor.getAllValues().get(0)).getID().toString());
    // submitDAG
    DAGPlan dagPlan = DAGPlan.getDefaultInstance();
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    dagClientHandler.submitDAG(dagPlan, localResources);
    verify(mockDagAM).submitDAGToAppMaster(dagPlan, localResources);
    // shutdown
    dagClientHandler.shutdownAM();
    verify(mockDagAM).shutdownTezAM(contains("Received message to shutdown AM from"));
}