Examples with TezDAGID org.apache.tez.dag.records.TezDAGID used on opensource projects

Example 36 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project tez by apache.

the class TestStateChangeNotifier method testUnregister.

@Test(timeout = 5000)
public void testUnregister() {
    TezDAGID dagId = TezDAGID.getInstance("1", 1, 1);
    Vertex v1 = createMockVertex(dagId, 1);
    DAG dag = createMockDag(dagId, v1);
    StateChangeNotifierForTest tracker = new StateChangeNotifierForTest(dag);
    VertexStateUpdateListener mockListener = mock(VertexStateUpdateListener.class);
    tracker.registerForVertexUpdates(v1.getName(), null, mockListener);
    List<VertexState> expectedStates = Lists.newArrayList(VertexState.RUNNING, VertexState.SUCCEEDED, VertexState.FAILED, VertexState.KILLED, VertexState.RUNNING, VertexState.SUCCEEDED);
    int count = 0;
    int numExpectedEvents = 3;
    for (VertexState state : expectedStates) {
        if (count == numExpectedEvents) {
            tracker.unregisterForVertexUpdates(v1.getName(), mockListener);
        }
        notifyTracker(tracker, v1, state);
        count++;
    }
    ArgumentCaptor<VertexStateUpdate> argumentCaptor = ArgumentCaptor.forClass(VertexStateUpdate.class);
    verify(mockListener, times(numExpectedEvents)).onStateUpdated(argumentCaptor.capture());
    List<VertexStateUpdate> stateUpdatesSent = argumentCaptor.getAllValues();
    Iterator<VertexState> expectedStateIter = expectedStates.iterator();
    for (int i = 0; i < numExpectedEvents; i++) {
        assertEquals(expectedStateIter.next(), stateUpdatesSent.get(i).getVertexState());
    }
}

Example 37 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project tez by apache.

the class Utils method processNonFatalServiceErrorReport.

public static void processNonFatalServiceErrorReport(String entityString, ServicePluginError servicePluginError, String diagnostics, DagInfo dagInfo, AppContext appContext, String componentName) {
    String message = "Error reported by " + componentName + " [" + entityString + "][" + servicePluginError + "] " + (diagnostics == null ? "" : diagnostics);
    if (dagInfo != null) {
        DAG dag = appContext.getCurrentDAG();
        if (dag != null && dag.getID().getId() == dagInfo.getIndex()) {
            TezDAGID dagId = dag.getID();
            // Send a kill message only if it is the same dag.
            LOG.warn(message + ", Failing dag: [" + dagInfo.getName() + ", " + dagId + "]");
            sendEvent(appContext, new DAGEventTerminateDag(dagId, DAGTerminationCause.SERVICE_PLUGIN_ERROR, message));
        }
    } else {
        LOG.warn("No current dag name provided. Not acting on " + message);
    }
}

Example 38 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project tez by apache.

the class ATSHistoryLoggingService method getDomainForEvent.

private String getDomainForEvent(DAGHistoryEvent event) {
    String domainId = sessionDomainId;
    if (historyACLPolicyManager == null) {
        return domainId;
    }
    TezDAGID dagId = event.getDagID();
    HistoryEvent historyEvent = event.getHistoryEvent();
    if (dagId == null || !HistoryEventType.isDAGSpecificEvent(historyEvent.getEventType())) {
        return domainId;
    }
    if (dagDomainIdMap.containsKey(dagId)) {
        // If we already have the domain for the dag id return it
        domainId = dagDomainIdMap.get(dagId);
        // Cleanup if this is the last event.
        if (historyEvent.getEventType() == HistoryEventType.DAG_FINISHED) {
            dagDomainIdMap.remove(dagId);
        }
    } else if (HistoryEventType.DAG_SUBMITTED == historyEvent.getEventType() || HistoryEventType.DAG_RECOVERED == historyEvent.getEventType()) {
        // In case this is the first event for the dag, create and populate dag domain.
        Configuration conf;
        DAGPlan dagPlan;
        if (HistoryEventType.DAG_SUBMITTED == historyEvent.getEventType()) {
            conf = ((DAGSubmittedEvent) historyEvent).getConf();
            dagPlan = ((DAGSubmittedEvent) historyEvent).getDAGPlan();
        } else {
            conf = appContext.getCurrentDAG().getConf();
            dagPlan = appContext.getCurrentDAG().getJobPlan();
        }
        domainId = createDagDomain(conf, dagPlan, dagId);
        // createDagDomain updates skippedDAGs so another check here.
        if (skippedDAGs.contains(dagId)) {
            return null;
        }
        dagDomainIdMap.put(dagId, domainId);
    }
    return domainId;
}

Example 39 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project tez by apache.

the class TestATSHistoryLoggingService method testNonSessionDomainsFailed.

@Test(timeout = 10000)
public void testNonSessionDomainsFailed() throws Exception {
    when(historyACLPolicyManager.setupSessionACLs((Configuration) any(), (ApplicationId) any())).thenThrow(new IOException());
    atsHistoryLoggingService.start();
    verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration) any(), (ApplicationId) any());
    // Send the event and wait for completion.
    TezDAGID dagId1 = TezDAGID.getInstance(appId, 0);
    for (DAGHistoryEvent event : makeHistoryEvents(dagId1, atsHistoryLoggingService)) {
        atsHistoryLoggingService.handle(event);
    }
    while (!atsHistoryLoggingService.eventQueue.isEmpty()) {
        Thread.sleep(1000);
    }
    // No dag domain were created.
    verify(historyACLPolicyManager, times(0)).setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any());
    // All calls made with session domain id.
    verify(historyACLPolicyManager, times(0)).updateTimelineEntityDomain(any(), eq("session-id"));
    Assert.assertEquals(0, atsEntitiesCounter);
}

Example 40 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project tez by apache.

the class TestATSHistoryLoggingService method testSessionDomainsDagFailed.

@Test(timeout = 10000)
public void testSessionDomainsDagFailed() throws Exception {
    when(historyACLPolicyManager.setupSessionACLs((Configuration) any(), (ApplicationId) any())).thenReturn(Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID, "session-domain"));
    when(historyACLPolicyManager.setupSessionDAGACLs((Configuration) any(), (ApplicationId) any(), eq("0"), (DAGAccessControls) any())).thenThrow(new IOException());
    when(appContext.isSession()).thenReturn(true);
    atsHistoryLoggingService.start();
    verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration) any(), (ApplicationId) any());
    // Send the event and wait for completion.
    TezDAGID dagId1 = TezDAGID.getInstance(appId, 0);
    for (DAGHistoryEvent event : makeHistoryEvents(dagId1, atsHistoryLoggingService)) {
        atsHistoryLoggingService.handle(event);
    }
    Thread.sleep(2500);
    while (!atsHistoryLoggingService.eventQueue.isEmpty()) {
        Thread.sleep(100);
    }
    // DAG domain was called once.
    verify(historyACLPolicyManager, times(1)).setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any());
    // All calls made with session domain id.
    verify(historyACLPolicyManager, times(1)).updateTimelineEntityDomain(any(), eq("session-domain"));
    verify(historyACLPolicyManager, times(1)).updateTimelineEntityDomain(any(), (String) any());
    Assert.assertEquals(1, atsEntitiesCounter);
}