Search in sources :

Example 61 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestATSV15HistoryLoggingService method testNonSessionDomains.

@Test
public void testNonSessionDomains() throws Exception {
    ATSV15HistoryLoggingService service = createService(-1);
    HistoryACLPolicyManager historyACLPolicyManager = mock(HistoryACLPolicyManager.class);
    service.historyACLPolicyManager = historyACLPolicyManager;
    when(historyACLPolicyManager.setupSessionACLs((Configuration) any(), eq(appId))).thenReturn(Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID, "session-id"));
    service.start();
    verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration) any(), eq(appId));
    // Send the event and wait for completion.
    TezDAGID dagId1 = TezDAGID.getInstance(appId, 0);
    for (DAGHistoryEvent event : makeHistoryEvents(dagId1, service)) {
        service.handle(event);
    }
    while (!service.eventQueue.isEmpty()) {
        Thread.sleep(100);
    }
    // No dag domain were created.
    verify(historyACLPolicyManager, times(0)).setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any());
    // All calls made with session domain id.
    // NOTE: Expect 6 invocations for 5 history events because DAG_SUBMITTED becomes two separate timeline events.
    verify(historyACLPolicyManager, times(6)).updateTimelineEntityDomain(any(), eq("session-id"));
    assertTrue(entityLog.size() > 0);
    service.stop();
}
Also used : HistoryACLPolicyManager(org.apache.tez.common.security.HistoryACLPolicyManager) TezDAGID(org.apache.tez.dag.records.TezDAGID) DAGHistoryEvent(org.apache.tez.dag.history.DAGHistoryEvent) Test(org.junit.Test)

Example 62 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestATSV15HistoryLoggingService method testNonSessionDomainsAclNull.

@Test
public void testNonSessionDomainsAclNull() throws Exception {
    ATSV15HistoryLoggingService service = createService(-1);
    HistoryACLPolicyManager historyACLPolicyManager = mock(HistoryACLPolicyManager.class);
    service.historyACLPolicyManager = historyACLPolicyManager;
    when(historyACLPolicyManager.setupSessionACLs((Configuration) any(), eq(appId))).thenReturn(null);
    service.start();
    verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration) any(), eq(appId));
    // Send the event and wait for completion.
    TezDAGID dagId1 = TezDAGID.getInstance(appId, 0);
    for (DAGHistoryEvent event : makeHistoryEvents(dagId1, service)) {
        service.handle(event);
    }
    while (!service.eventQueue.isEmpty()) {
        Thread.sleep(100);
    }
    // No dag domain were created.
    verify(historyACLPolicyManager, times(0)).setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any());
    // No domain updates but history logging happened.
    verify(historyACLPolicyManager, times(0)).updateTimelineEntityDomain(any(), (String) any());
    assertTrue(entityLog.size() > 0);
    service.stop();
}
Also used : HistoryACLPolicyManager(org.apache.tez.common.security.HistoryACLPolicyManager) TezDAGID(org.apache.tez.dag.records.TezDAGID) DAGHistoryEvent(org.apache.tez.dag.history.DAGHistoryEvent) Test(org.junit.Test)

Example 63 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestATSHistoryWithACLs method testDagLoggingEnabled.

/**
 * use mini cluster to verify data do push to ats when
 * the dag logging flag in dagsubmitted event is set on
 * @throws Exception
 */
@Test(timeout = 50000)
public void testDagLoggingEnabled() throws Exception {
    ATSHistoryLoggingService historyLoggingService;
    historyLoggingService = ReflectionUtils.createClazzInstance(ATSHistoryLoggingService.class.getName());
    AppContext appContext = mock(AppContext.class);
    when(appContext.getApplicationID()).thenReturn(ApplicationId.newInstance(0, 1));
    historyLoggingService.setAppContext(appContext);
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    String viewAcls = "nobody nobody_group";
    tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
    tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName());
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
    historyLoggingService.init(tezConf);
    historyLoggingService.start();
    ApplicationId appId = ApplicationId.newInstance(100l, 1);
    TezDAGID tezDAGID = TezDAGID.getInstance(appId, 11);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    DAGPlan dagPlan = DAGPlan.newBuilder().setName("DAGPlanMock").build();
    DAGSubmittedEvent submittedEvent = new DAGSubmittedEvent(tezDAGID, 1, dagPlan, appAttemptId, null, "usr", tezConf, null, null);
    submittedEvent.setHistoryLoggingEnabled(true);
    DAGHistoryEvent event = new DAGHistoryEvent(tezDAGID, submittedEvent);
    historyLoggingService.handle(new DAGHistoryEvent(tezDAGID, submittedEvent));
    Thread.sleep(1000l);
    String url = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_DAG_ID/" + event.getDagID();
    Client client = new Client();
    WebResource resource = client.resource(url);
    ClientResponse response = resource.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(200, response.getStatus());
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    TimelineEntity entity = response.getEntity(TimelineEntity.class);
    assertEquals(entity.getEntityType(), "TEZ_DAG_ID");
    assertEquals(entity.getEvents().get(0).getEventType(), HistoryEventType.DAG_SUBMITTED.toString());
}
Also used : Path(org.apache.hadoop.fs.Path) ClientResponse(com.sun.jersey.api.client.ClientResponse) AppContext(org.apache.tez.dag.app.AppContext) DAGHistoryEvent(org.apache.tez.dag.history.DAGHistoryEvent) WebResource(com.sun.jersey.api.client.WebResource) ApplicationAttemptId(org.apache.hadoop.yarn.api.records.ApplicationAttemptId) TimelineEntity(org.apache.hadoop.yarn.api.records.timeline.TimelineEntity) ATSHistoryLoggingService(org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService) DAGPlan(org.apache.tez.dag.api.records.DAGProtos.DAGPlan) TezDAGID(org.apache.tez.dag.records.TezDAGID) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) DAGClient(org.apache.tez.dag.api.client.DAGClient) TezClient(org.apache.tez.client.TezClient) Client(com.sun.jersey.api.client.Client) TezConfiguration(org.apache.tez.dag.api.TezConfiguration) DAGSubmittedEvent(org.apache.tez.dag.history.events.DAGSubmittedEvent) Test(org.junit.Test)

Example 64 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TaskImpl method logJobHistoryTaskStartedEvent.

protected void logJobHistoryTaskStartedEvent() {
    TaskStartedEvent startEvt = new TaskStartedEvent(taskId, getVertex().getName(), scheduledTime, getLaunchTime());
    this.appContext.getHistoryHandler().handle(new DAGHistoryEvent(taskId.getVertexID().getDAGId(), startEvt));
}
Also used : DAGHistoryEvent(org.apache.tez.dag.history.DAGHistoryEvent) TaskStartedEvent(org.apache.tez.dag.history.events.TaskStartedEvent)

Example 65 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class VertexImpl method logJobHistoryVertexCompletedHelper.

private void logJobHistoryVertexCompletedHelper(VertexState finalState, long finishTime, String diagnostics, TezCounters counters) throws IOException {
    Map<String, Integer> taskStats = new HashMap<String, Integer>();
    taskStats.put(ATSConstants.NUM_COMPLETED_TASKS, completedTaskCount);
    taskStats.put(ATSConstants.NUM_SUCCEEDED_TASKS, succeededTaskCount);
    taskStats.put(ATSConstants.NUM_FAILED_TASKS, failedTaskCount);
    taskStats.put(ATSConstants.NUM_KILLED_TASKS, killedTaskCount);
    taskStats.put(ATSConstants.NUM_FAILED_TASKS_ATTEMPTS, failedTaskAttemptCount.get());
    taskStats.put(ATSConstants.NUM_KILLED_TASKS_ATTEMPTS, killedTaskAttemptCount.get());
    VertexFinishedEvent finishEvt = new VertexFinishedEvent(vertexId, vertexName, numTasks, initTimeRequested, initedTime, startTimeRequested, startedTime, finishTime, finalState, diagnostics, counters, getVertexStats(), taskStats, servicePluginInfo);
    this.appContext.getHistoryHandler().handleCriticalEvent(new DAGHistoryEvent(getDAGId(), finishEvt));
}
Also used : AtomicInteger(java.util.concurrent.atomic.AtomicInteger) LinkedHashMap(java.util.LinkedHashMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) DAGHistoryEvent(org.apache.tez.dag.history.DAGHistoryEvent) VertexFinishedEvent(org.apache.tez.dag.history.events.VertexFinishedEvent)

Aggregations

DAGHistoryEvent (org.apache.tez.dag.history.DAGHistoryEvent)81 TezDAGID (org.apache.tez.dag.records.TezDAGID)38 Test (org.junit.Test)33 DAGSubmittedEvent (org.apache.tez.dag.history.events.DAGSubmittedEvent)21 IOException (java.io.IOException)18 Configuration (org.apache.hadoop.conf.Configuration)18 DAGPlan (org.apache.tez.dag.api.records.DAGProtos.DAGPlan)18 ApplicationId (org.apache.hadoop.yarn.api.records.ApplicationId)17 Path (org.apache.hadoop.fs.Path)15 SystemClock (org.apache.hadoop.yarn.util.SystemClock)14 DAGRecoveryData (org.apache.tez.dag.app.RecoveryParser.DAGRecoveryData)13 DAGStartedEvent (org.apache.tez.dag.history.events.DAGStartedEvent)11 RecoveryService (org.apache.tez.dag.history.recovery.RecoveryService)11 TezVertexID (org.apache.tez.dag.records.TezVertexID)10 TaskStartedEvent (org.apache.tez.dag.history.events.TaskStartedEvent)7 ApplicationAttemptId (org.apache.hadoop.yarn.api.records.ApplicationAttemptId)6 TimelineEntity (org.apache.hadoop.yarn.api.records.timeline.TimelineEntity)6 TezConfiguration (org.apache.tez.dag.api.TezConfiguration)6 DAGFinishedEvent (org.apache.tez.dag.history.events.DAGFinishedEvent)6 VertexFinishedEvent (org.apache.tez.dag.history.events.VertexFinishedEvent)6