Examples with DAGHistoryEvent org.apache.tez.dag.history.DAGHistoryEvent used on opensource projects

Example 26 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestATSHistoryLoggingService method testSessionDomainsDagFailed.

@Test(timeout = 10000)
public void testSessionDomainsDagFailed() throws Exception {
    when(historyACLPolicyManager.setupSessionACLs((Configuration) any(), (ApplicationId) any())).thenReturn(Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID, "session-domain"));
    when(historyACLPolicyManager.setupSessionDAGACLs((Configuration) any(), (ApplicationId) any(), eq("0"), (DAGAccessControls) any())).thenThrow(new IOException());
    when(appContext.isSession()).thenReturn(true);
    atsHistoryLoggingService.start();
    verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration) any(), (ApplicationId) any());
    // Send the event and wait for completion.
    TezDAGID dagId1 = TezDAGID.getInstance(appId, 0);
    for (DAGHistoryEvent event : makeHistoryEvents(dagId1, atsHistoryLoggingService)) {
        atsHistoryLoggingService.handle(event);
    }
    Thread.sleep(2500);
    while (!atsHistoryLoggingService.eventQueue.isEmpty()) {
        Thread.sleep(100);
    }
    // DAG domain was called once.
    verify(historyACLPolicyManager, times(1)).setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any());
    // All calls made with session domain id.
    verify(historyACLPolicyManager, times(1)).updateTimelineEntityDomain(any(), eq("session-domain"));
    verify(historyACLPolicyManager, times(1)).updateTimelineEntityDomain(any(), (String) any());
    Assert.assertEquals(1, atsEntitiesCounter);
}

Example 27 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestATSHistoryLoggingService method testATSEventBatching.

@Test(timeout = 20000)
public void testATSEventBatching() {
    atsHistoryLoggingService.start();
    TezDAGID tezDAGID = TezDAGID.getInstance(ApplicationId.newInstance(100l, 1), 1);
    DAGHistoryEvent historyEvent = new DAGHistoryEvent(tezDAGID, new DAGStartedEvent(tezDAGID, 1001l, "user1", "dagName1"));
    for (int i = 0; i < 100; ++i) {
        atsHistoryLoggingService.handle(historyEvent);
    }
    try {
        Thread.sleep(1000l);
    } catch (InterruptedException e) {
    // Do nothing
    }
    LOG.info("ATS entitiesSent=" + atsEntitiesCounter + ", timelineInvocations=" + atsInvokeCounter);
    Assert.assertTrue(atsEntitiesCounter > atsInvokeCounter);
    Assert.assertEquals(atsEntitiesCounter / 2, atsInvokeCounter);
}

Example 28 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestATSHistoryLoggingService method testNonSessionDomainsAclNull.

@Test(timeout = 10000)
public void testNonSessionDomainsAclNull() throws Exception {
    when(historyACLPolicyManager.setupSessionACLs((Configuration) any(), (ApplicationId) any())).thenReturn(null);
    atsHistoryLoggingService.start();
    verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration) any(), (ApplicationId) any());
    // Send the event and wait for completion.
    TezDAGID dagId1 = TezDAGID.getInstance(appId, 0);
    for (DAGHistoryEvent event : makeHistoryEvents(dagId1, atsHistoryLoggingService)) {
        atsHistoryLoggingService.handle(event);
    }
    Thread.sleep(2500);
    while (!atsHistoryLoggingService.eventQueue.isEmpty()) {
        Thread.sleep(100);
    }
    // No dag domain were created.
    verify(historyACLPolicyManager, times(0)).setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any());
    // All calls made with session domain id.
    verify(historyACLPolicyManager, times(0)).updateTimelineEntityDomain(any(), eq("session-id"));
    Assert.assertEquals(6, atsEntitiesCounter);
}

Example 29 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestATSHistoryLoggingService method makeHistoryEvents.

private List<DAGHistoryEvent> makeHistoryEvents(TezDAGID dagId, ATSHistoryLoggingService service) {
    List<DAGHistoryEvent> historyEvents = new ArrayList<>();
    long time = System.currentTimeMillis();
    Configuration conf = new Configuration(service.getConfig());
    historyEvents.add(new DAGHistoryEvent(null, new AMStartedEvent(attemptId, time, "user")));
    historyEvents.add(new DAGHistoryEvent(dagId, new DAGSubmittedEvent(dagId, time, DAGPlan.getDefaultInstance(), attemptId, null, "user", conf, null, "default")));
    TezVertexID vertexID = TezVertexID.getInstance(dagId, 1);
    historyEvents.add(new DAGHistoryEvent(dagId, new VertexStartedEvent(vertexID, time, time)));
    TezTaskID tezTaskID = TezTaskID.getInstance(vertexID, 1);
    historyEvents.add(new DAGHistoryEvent(dagId, new TaskStartedEvent(tezTaskID, "test", time, time)));
    historyEvents.add(new DAGHistoryEvent(dagId, new TaskAttemptStartedEvent(TezTaskAttemptID.getInstance(tezTaskID, 1), "test", time, ContainerId.newContainerId(attemptId, 1), NodeId.newInstance("localhost", 8765), null, null, null)));
    return historyEvents;
}

Example 30 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestATSHistoryLoggingService method testSessionDomainsFailed.

@Test(timeout = 10000)
public void testSessionDomainsFailed() throws Exception {
    when(historyACLPolicyManager.setupSessionACLs((Configuration) any(), (ApplicationId) any())).thenThrow(new IOException());
    when(historyACLPolicyManager.setupSessionDAGACLs((Configuration) any(), (ApplicationId) any(), eq("0"), (DAGAccessControls) any())).thenReturn(Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_DAG_DOMAIN_ID, "dag-domain"));
    when(appContext.isSession()).thenReturn(true);
    atsHistoryLoggingService.start();
    verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration) any(), (ApplicationId) any());
    // Send the event and wait for completion.
    TezDAGID dagId1 = TezDAGID.getInstance(appId, 0);
    for (DAGHistoryEvent event : makeHistoryEvents(dagId1, atsHistoryLoggingService)) {
        atsHistoryLoggingService.handle(event);
    }
    while (!atsHistoryLoggingService.eventQueue.isEmpty()) {
        Thread.sleep(1000);
    }
    // No dag domain were created.
    verify(historyACLPolicyManager, times(0)).setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any());
    // No calls were made for domains.
    verify(historyACLPolicyManager, times(0)).updateTimelineEntityDomain(any(), (String) any());
    Assert.assertEquals(0, atsEntitiesCounter);
}