Examples with DAGHistoryEvent org.apache.tez.dag.history.DAGHistoryEvent used on opensource projects

Example 56 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class ATSV15HistoryLoggingService method serviceStart.

@Override
public void serviceStart() {
    if (!historyLoggingEnabled || timelineClient == null) {
        return;
    }
    timelineClient.start();
    // create a session domain id, if it fails then disable history logging.
    try {
        sessionDomainId = createSessionDomain();
    } catch (HistoryACLPolicyException | IOException e) {
        LOG.warn("Could not setup history acls, disabling history logging.", e);
        historyLoggingEnabled = false;
        return;
    }
    eventHandlingThread = new Thread(new Runnable() {

        @Override
        public void run() {
            boolean interrupted = false;
            TezUtilsInternal.setHadoopCallerContext(appContext.getHadoopShim(), appContext.getApplicationID());
            while (!stopped.get() && !Thread.currentThread().isInterrupted() && !interrupted) {
                // Log the size of the event-queue every so often.
                if (eventCounter != 0 && eventCounter % 1000 == 0) {
                    if (eventsProcessed != 0 && !eventQueue.isEmpty()) {
                        LOG.info("Event queue stats" + ", eventsProcessedSinceLastUpdate=" + eventsProcessed + ", eventQueueSize=" + eventQueue.size());
                    }
                    eventCounter = 0;
                    eventsProcessed = 0;
                } else {
                    ++eventCounter;
                }
                synchronized (lock) {
                    try {
                        DAGHistoryEvent event = eventQueue.poll(maxPollingTimeMillis, TimeUnit.MILLISECONDS);
                        if (event == null) {
                            continue;
                        }
                        if (!isValidEvent(event)) {
                            continue;
                        }
                        try {
                            handleEvents(event);
                            eventsProcessed += 1;
                        } catch (Exception e) {
                            LOG.warn("Error handling events", e);
                        }
                    } catch (InterruptedException e) {
                        // Finish processing events and then return
                        interrupted = true;
                    }
                }
            }
        }
    }, "HistoryEventHandlingThread");
    eventHandlingThread.start();
}

Example 57 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class ATSV15HistoryLoggingService method serviceStop.

@Override
public void serviceStop() {
    LOG.info("Stopping ATSService" + ", eventQueueBacklog=" + eventQueue.size());
    stopped.set(true);
    if (eventHandlingThread != null) {
        eventHandlingThread.interrupt();
    }
    try {
        TezUtilsInternal.setHadoopCallerContext(appContext.getHadoopShim(), appContext.getApplicationID());
        synchronized (lock) {
            if (!eventQueue.isEmpty()) {
                LOG.warn("ATSService being stopped" + ", eventQueueBacklog=" + eventQueue.size() + ", maxTimeLeftToFlush=" + maxTimeToWaitOnShutdown + ", waitForever=" + waitForeverOnShutdown);
                long startTime = appContext.getClock().getTime();
                long endTime = startTime + maxTimeToWaitOnShutdown;
                while (waitForeverOnShutdown || (endTime >= appContext.getClock().getTime())) {
                    try {
                        DAGHistoryEvent event = eventQueue.poll(maxPollingTimeMillis, TimeUnit.MILLISECONDS);
                        if (event == null) {
                            LOG.info("Event queue empty, stopping ATS Service");
                            break;
                        }
                        if (!isValidEvent(event)) {
                            continue;
                        }
                        try {
                            handleEvents(event);
                        } catch (Exception e) {
                            LOG.warn("Error handling event", e);
                        }
                    } catch (InterruptedException e) {
                        LOG.info("ATSService interrupted while shutting down. Exiting." + " EventQueueBacklog=" + eventQueue.size());
                    }
                }
            }
        }
    } finally {
        appContext.getHadoopShim().clearHadoopCallerContext();
    }
    if (!eventQueue.isEmpty()) {
        LOG.warn("Did not finish flushing eventQueue before stopping ATSService" + ", eventQueueBacklog=" + eventQueue.size());
    }
    if (timelineClient != null) {
        timelineClient.stop();
    }
    if (historyACLPolicyManager != null) {
        historyACLPolicyManager.close();
    }
}

Example 58 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestATSV15HistoryLoggingService method testSessionDomainsDagFailed.

@Test
public void testSessionDomainsDagFailed() throws Exception {
    ATSV15HistoryLoggingService service = createService(-1);
    when(appContext.isSession()).thenReturn(true);
    HistoryACLPolicyManager historyACLPolicyManager = mock(HistoryACLPolicyManager.class);
    service.historyACLPolicyManager = historyACLPolicyManager;
    when(historyACLPolicyManager.setupSessionACLs((Configuration) any(), eq(appId))).thenReturn(Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID, "session-id"));
    service.start();
    // Verify that the session domain creation was called.
    verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration) any(), eq(appId));
    // Mock dag domain creation.
    when(historyACLPolicyManager.setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any())).thenThrow(new IOException());
    // Send the event and wait for completion.
    TezDAGID dagId1 = TezDAGID.getInstance(appId, 0);
    for (DAGHistoryEvent event : makeHistoryEvents(dagId1, service)) {
        service.handle(event);
    }
    while (!service.eventQueue.isEmpty()) {
        Thread.sleep(100);
    }
    // Verify dag domain creation was called.
    verify(historyACLPolicyManager, times(1)).setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any());
    // AM events sent, dag events are not sent.
    verify(historyACLPolicyManager, times(1)).updateTimelineEntityDomain(any(), eq("session-id"));
    verify(historyACLPolicyManager, times(0)).updateTimelineEntityDomain(any(), eq("dag-id"));
    assertEquals(1, entityLog.size());
    service.stop();
}

Example 59 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestATSV15HistoryLoggingService method testSessionDomains.

@Test
public void testSessionDomains() throws Exception {
    ATSV15HistoryLoggingService service = createService(-1);
    when(appContext.isSession()).thenReturn(true);
    HistoryACLPolicyManager historyACLPolicyManager = mock(HistoryACLPolicyManager.class);
    service.historyACLPolicyManager = historyACLPolicyManager;
    when(historyACLPolicyManager.setupSessionACLs((Configuration) any(), eq(appId))).thenReturn(Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID, "session-id"));
    service.start();
    // Verify that the session domain was created.
    verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration) any(), eq(appId));
    // Mock dag domain creation.
    when(historyACLPolicyManager.setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any())).thenReturn(Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_DAG_DOMAIN_ID, "dag-id"));
    // Send the event and wait for completion.
    TezDAGID dagId1 = TezDAGID.getInstance(appId, 0);
    for (DAGHistoryEvent event : makeHistoryEvents(dagId1, service)) {
        service.handle(event);
    }
    while (!service.eventQueue.isEmpty()) {
        Thread.sleep(100);
    }
    // Verify dag domain was created.
    verify(historyACLPolicyManager, times(1)).setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any());
    // calls were made with correct domain ids.
    verify(historyACLPolicyManager, times(1)).updateTimelineEntityDomain(any(), eq("session-id"));
    verify(historyACLPolicyManager, times(5)).updateTimelineEntityDomain(any(), eq("dag-id"));
    service.stop();
}

Example 60 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestATSV15HistoryLoggingService method testDAGGroupingDefault.

@Test(timeout = 2000)
public void testDAGGroupingDefault() throws Exception {
    ATSV15HistoryLoggingService service = createService(-1);
    service.start();
    TezDAGID dagId1 = TezDAGID.getInstance(appId, 0);
    for (DAGHistoryEvent event : makeHistoryEvents(dagId1, service)) {
        service.handle(event);
    }
    while (!service.eventQueue.isEmpty()) {
        Thread.sleep(100);
    }
    assertEquals(2, entityLog.size());
    List<TimelineEntity> amEvents = entityLog.get(TimelineEntityGroupId.newInstance(appId, appId.toString()));
    assertNotNull(amEvents);
    assertEquals(1, amEvents.size());
    List<TimelineEntity> nonGroupedDagEvents = entityLog.get(TimelineEntityGroupId.newInstance(appId, dagId1.toString()));
    assertNotNull(nonGroupedDagEvents);
    assertEquals(5, nonGroupedDagEvents.size());
    service.stop();
}