use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.
the class TestATSV15HistoryLoggingService method testNonSessionDomains.
@Test
public void testNonSessionDomains() throws Exception {
ATSV15HistoryLoggingService service = createService(-1);
HistoryACLPolicyManager historyACLPolicyManager = mock(HistoryACLPolicyManager.class);
service.historyACLPolicyManager = historyACLPolicyManager;
when(historyACLPolicyManager.setupSessionACLs((Configuration) any(), eq(appId))).thenReturn(Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID, "session-id"));
service.start();
verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration) any(), eq(appId));
// Send the event and wait for completion.
TezDAGID dagId1 = TezDAGID.getInstance(appId, 0);
for (DAGHistoryEvent event : makeHistoryEvents(dagId1, service)) {
service.handle(event);
}
while (!service.eventQueue.isEmpty()) {
Thread.sleep(100);
}
// No dag domain were created.
verify(historyACLPolicyManager, times(0)).setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any());
// All calls made with session domain id.
// NOTE: Expect 6 invocations for 5 history events because DAG_SUBMITTED becomes two separate timeline events.
verify(historyACLPolicyManager, times(6)).updateTimelineEntityDomain(any(), eq("session-id"));
assertTrue(entityLog.size() > 0);
service.stop();
}
use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.
the class TestATSV15HistoryLoggingService method testNonSessionDomainsAclNull.
@Test
public void testNonSessionDomainsAclNull() throws Exception {
ATSV15HistoryLoggingService service = createService(-1);
HistoryACLPolicyManager historyACLPolicyManager = mock(HistoryACLPolicyManager.class);
service.historyACLPolicyManager = historyACLPolicyManager;
when(historyACLPolicyManager.setupSessionACLs((Configuration) any(), eq(appId))).thenReturn(null);
service.start();
verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration) any(), eq(appId));
// Send the event and wait for completion.
TezDAGID dagId1 = TezDAGID.getInstance(appId, 0);
for (DAGHistoryEvent event : makeHistoryEvents(dagId1, service)) {
service.handle(event);
}
while (!service.eventQueue.isEmpty()) {
Thread.sleep(100);
}
// No dag domain were created.
verify(historyACLPolicyManager, times(0)).setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any());
// No domain updates but history logging happened.
verify(historyACLPolicyManager, times(0)).updateTimelineEntityDomain(any(), (String) any());
assertTrue(entityLog.size() > 0);
service.stop();
}
use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.
the class TestATSHistoryWithACLs method testDagLoggingEnabled.
/**
* use mini cluster to verify data do push to ats when
* the dag logging flag in dagsubmitted event is set on
* @throws Exception
*/
@Test(timeout = 50000)
public void testDagLoggingEnabled() throws Exception {
ATSHistoryLoggingService historyLoggingService;
historyLoggingService = ReflectionUtils.createClazzInstance(ATSHistoryLoggingService.class.getName());
AppContext appContext = mock(AppContext.class);
when(appContext.getApplicationID()).thenReturn(ApplicationId.newInstance(0, 1));
historyLoggingService.setAppContext(appContext);
TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
String viewAcls = "nobody nobody_group";
tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName());
Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
remoteFs.mkdirs(remoteStagingDir);
tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
historyLoggingService.init(tezConf);
historyLoggingService.start();
ApplicationId appId = ApplicationId.newInstance(100l, 1);
TezDAGID tezDAGID = TezDAGID.getInstance(appId, 11);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
DAGPlan dagPlan = DAGPlan.newBuilder().setName("DAGPlanMock").build();
DAGSubmittedEvent submittedEvent = new DAGSubmittedEvent(tezDAGID, 1, dagPlan, appAttemptId, null, "usr", tezConf, null, null);
submittedEvent.setHistoryLoggingEnabled(true);
DAGHistoryEvent event = new DAGHistoryEvent(tezDAGID, submittedEvent);
historyLoggingService.handle(new DAGHistoryEvent(tezDAGID, submittedEvent));
Thread.sleep(1000l);
String url = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_DAG_ID/" + event.getDagID();
Client client = new Client();
WebResource resource = client.resource(url);
ClientResponse response = resource.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(200, response.getStatus());
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
TimelineEntity entity = response.getEntity(TimelineEntity.class);
assertEquals(entity.getEntityType(), "TEZ_DAG_ID");
assertEquals(entity.getEvents().get(0).getEventType(), HistoryEventType.DAG_SUBMITTED.toString());
}
use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.
the class TaskImpl method logJobHistoryTaskStartedEvent.
protected void logJobHistoryTaskStartedEvent() {
TaskStartedEvent startEvt = new TaskStartedEvent(taskId, getVertex().getName(), scheduledTime, getLaunchTime());
this.appContext.getHistoryHandler().handle(new DAGHistoryEvent(taskId.getVertexID().getDAGId(), startEvt));
}
use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.
the class VertexImpl method logJobHistoryVertexCompletedHelper.
private void logJobHistoryVertexCompletedHelper(VertexState finalState, long finishTime, String diagnostics, TezCounters counters) throws IOException {
Map<String, Integer> taskStats = new HashMap<String, Integer>();
taskStats.put(ATSConstants.NUM_COMPLETED_TASKS, completedTaskCount);
taskStats.put(ATSConstants.NUM_SUCCEEDED_TASKS, succeededTaskCount);
taskStats.put(ATSConstants.NUM_FAILED_TASKS, failedTaskCount);
taskStats.put(ATSConstants.NUM_KILLED_TASKS, killedTaskCount);
taskStats.put(ATSConstants.NUM_FAILED_TASKS_ATTEMPTS, failedTaskAttemptCount.get());
taskStats.put(ATSConstants.NUM_KILLED_TASKS_ATTEMPTS, killedTaskAttemptCount.get());
VertexFinishedEvent finishEvt = new VertexFinishedEvent(vertexId, vertexName, numTasks, initTimeRequested, initedTime, startTimeRequested, startedTime, finishTime, finalState, diagnostics, counters, getVertexStats(), taskStats, servicePluginInfo);
this.appContext.getHistoryHandler().handleCriticalEvent(new DAGHistoryEvent(getDAGId(), finishEvt));
}
Aggregations