Examples with DAGHistoryEvent org.apache.tez.dag.history.DAGHistoryEvent used on opensource projects

Example 76 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class ATSHistoryLoggingService method handleEvents.

private void handleEvents(List<DAGHistoryEvent> events) {
    List<TimelineEntity> entities = new ArrayList<>(events.size());
    for (DAGHistoryEvent event : events) {
        String domainId = getDomainForEvent(event);
        // skippedDags is updated in the above call so check again.
        if (event.getDagID() != null && skippedDAGs.contains(event.getDagID())) {
            continue;
        }
        List<TimelineEntity> eventEntities = HistoryEventTimelineConversion.convertToTimelineEntities(event.getHistoryEvent());
        entities.addAll(eventEntities);
        if (historyACLPolicyManager != null && domainId != null && !domainId.isEmpty()) {
            for (TimelineEntity entity : eventEntities) {
                historyACLPolicyManager.updateTimelineEntityDomain(entity, domainId);
            }
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Sending event batch to Timeline, batchSize=" + events.size());
    }
    try {
        TimelinePutResponse response = timelineClient.putEntities(entities.toArray(new TimelineEntity[entities.size()]));
        if (response != null && !response.getErrors().isEmpty()) {
            int count = response.getErrors().size();
            for (int i = 0; i < count; ++i) {
                TimelinePutError err = response.getErrors().get(i);
                if (err.getErrorCode() != 0) {
                    LOG.warn("Could not post history event to ATS" + ", atsPutError=" + err.getErrorCode() + ", entityId=" + err.getEntityId());
                }
            }
        }
    // Do nothing additional, ATS client library should handle throttling
    // or auto-disable as needed
    } catch (Exception e) {
        LOG.warn("Could not handle history events", e);
    }
}

Example 77 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestATSHistoryLoggingService method testTimelineServiceDisable.

@Test(timeout = 20000)
public void testTimelineServiceDisable() throws Exception {
    atsHistoryLoggingService.start();
    ATSHistoryLoggingService atsHistoryLoggingService1;
    atsHistoryLoggingService1 = new ATSHistoryLoggingService();
    atsHistoryLoggingService1.setAppContext(appContext);
    atsHistoryLoggingService1.timelineClient = mock(TimelineClient.class);
    when(atsHistoryLoggingService1.timelineClient.putEntities(Matchers.<TimelineEntity[]>anyVararg())).thenAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            ++atsInvokeCounter;
            atsEntitiesCounter += invocation.getArguments().length;
            try {
                Thread.sleep(10l);
            } catch (InterruptedException e) {
            // do nothing
            }
            return null;
        }
    });
    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
    conf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName());
    atsHistoryLoggingService1.init(conf);
    atsHistoryLoggingService1.start();
    TezDAGID tezDAGID = TezDAGID.getInstance(ApplicationId.newInstance(100l, 1), 1);
    DAGHistoryEvent historyEvent = new DAGHistoryEvent(tezDAGID, new DAGStartedEvent(tezDAGID, 1001l, "user1", "dagName1"));
    for (int i = 0; i < 100; ++i) {
        atsHistoryLoggingService1.handle(historyEvent);
    }
    try {
        Thread.sleep(20l);
    } catch (InterruptedException e) {
    // Do nothing
    }
    LOG.info("ATS entitiesSent=" + atsEntitiesCounter + ", timelineInvocations=" + atsInvokeCounter);
    Assert.assertEquals(atsInvokeCounter, 0);
    Assert.assertEquals(atsEntitiesCounter, 0);
    Assert.assertNull(atsHistoryLoggingService1.timelineClient);
    atsHistoryLoggingService1.close();
}

Example 78 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestATSHistoryLoggingService method testSessionDomains.

@Test(timeout = 10000)
public void testSessionDomains() throws Exception {
    when(historyACLPolicyManager.setupSessionACLs((Configuration) any(), (ApplicationId) any())).thenReturn(Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID, "test-domain"));
    when(historyACLPolicyManager.setupSessionDAGACLs((Configuration) any(), (ApplicationId) any(), eq("0"), (DAGAccessControls) any())).thenReturn(Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_DAG_DOMAIN_ID, "dag-domain"));
    when(appContext.isSession()).thenReturn(true);
    atsHistoryLoggingService.start();
    verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration) any(), (ApplicationId) any());
    // Send the event and wait for completion.
    TezDAGID dagId1 = TezDAGID.getInstance(appId, 0);
    for (DAGHistoryEvent event : makeHistoryEvents(dagId1, atsHistoryLoggingService)) {
        atsHistoryLoggingService.handle(event);
    }
    Thread.sleep(2500);
    while (!atsHistoryLoggingService.eventQueue.isEmpty()) {
        Thread.sleep(100);
    }
    // No dag domain were created.
    verify(historyACLPolicyManager, times(1)).setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any());
    // All calls made with session domain id.
    verify(historyACLPolicyManager, times(1)).updateTimelineEntityDomain(any(), eq("test-domain"));
    verify(historyACLPolicyManager, times(5)).updateTimelineEntityDomain(any(), eq("dag-domain"));
}

Example 79 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestATSHistoryLoggingService method testATSHistoryLoggingServiceShutdown.

@Test(timeout = 20000)
public void testATSHistoryLoggingServiceShutdown() {
    atsHistoryLoggingService.start();
    TezDAGID tezDAGID = TezDAGID.getInstance(ApplicationId.newInstance(100l, 1), 1);
    DAGHistoryEvent historyEvent = new DAGHistoryEvent(tezDAGID, new DAGStartedEvent(tezDAGID, 1001l, "user1", "dagName1"));
    for (int i = 0; i < 100; ++i) {
        atsHistoryLoggingService.handle(historyEvent);
    }
    try {
        Thread.sleep(2500l);
    } catch (InterruptedException e) {
    // Do nothing
    }
    atsHistoryLoggingService.stop();
    LOG.info("ATS entitiesSent=" + atsEntitiesCounter + ", timelineInvocations=" + atsInvokeCounter);
    Assert.assertTrue(atsEntitiesCounter >= 4);
    Assert.assertTrue(atsEntitiesCounter < 20);
}

Example 80 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestATSHistoryLoggingService method testSessionDomainsAclNull.

@Test(timeout = 10000)
public void testSessionDomainsAclNull() throws Exception {
    when(historyACLPolicyManager.setupSessionACLs((Configuration) any(), (ApplicationId) any())).thenReturn(null);
    when(historyACLPolicyManager.setupSessionDAGACLs((Configuration) any(), (ApplicationId) any(), eq("0"), (DAGAccessControls) any())).thenReturn(null);
    when(appContext.isSession()).thenReturn(true);
    atsHistoryLoggingService.start();
    verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration) any(), (ApplicationId) any());
    // Send the event and wait for completion.
    TezDAGID dagId1 = TezDAGID.getInstance(appId, 0);
    for (DAGHistoryEvent event : makeHistoryEvents(dagId1, atsHistoryLoggingService)) {
        atsHistoryLoggingService.handle(event);
    }
    Thread.sleep(2500);
    while (!atsHistoryLoggingService.eventQueue.isEmpty()) {
        Thread.sleep(100);
    }
    // No dag domain were created.
    verify(historyACLPolicyManager, times(1)).setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any());
    // All calls made with session domain id.
    verify(historyACLPolicyManager, times(0)).updateTimelineEntityDomain(any(), (String) any());
    Assert.assertEquals(6, atsEntitiesCounter);
}