Examples with ATSHistoryLoggingService org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService used on opensource projects

Example 1 with ATSHistoryLoggingService

use of org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService in project tez by apache.

the class TestATSHistoryWithACLs method testDagLoggingDisabled.

/**
 * use mini cluster to verify data do not push to ats when the daglogging flag
 * in dagsubmittedevent is set off
 * @throws Exception
 */
@Test(timeout = 50000)
public void testDagLoggingDisabled() throws Exception {
    ATSHistoryLoggingService historyLoggingService;
    historyLoggingService = ReflectionUtils.createClazzInstance(ATSHistoryLoggingService.class.getName());
    AppContext appContext = mock(AppContext.class);
    when(appContext.getApplicationID()).thenReturn(ApplicationId.newInstance(0, 1));
    historyLoggingService.setAppContext(appContext);
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    String viewAcls = "nobody nobody_group";
    tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
    tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName());
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
    historyLoggingService.init(tezConf);
    historyLoggingService.start();
    ApplicationId appId = ApplicationId.newInstance(100l, 1);
    TezDAGID tezDAGID = TezDAGID.getInstance(appId, 100);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    DAGPlan dagPlan = DAGPlan.newBuilder().setName("DAGPlanMock").build();
    DAGSubmittedEvent submittedEvent = new DAGSubmittedEvent(tezDAGID, 1, dagPlan, appAttemptId, null, "usr", tezConf, null, null);
    submittedEvent.setHistoryLoggingEnabled(false);
    DAGHistoryEvent event = new DAGHistoryEvent(tezDAGID, submittedEvent);
    historyLoggingService.handle(new DAGHistoryEvent(tezDAGID, submittedEvent));
    Thread.sleep(1000l);
    String url = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_DAG_ID/" + event.getDagID();
    Client client = new Client();
    WebResource resource = client.resource(url);
    ClientResponse response = resource.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(404, response.getStatus());
}

Example 2 with ATSHistoryLoggingService

use of org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService in project tez by apache.

the class TestATSHistoryWithACLs method testDagLoggingEnabled.

/**
 * use mini cluster to verify data do push to ats when
 * the dag logging flag in dagsubmitted event is set on
 * @throws Exception
 */
@Test(timeout = 50000)
public void testDagLoggingEnabled() throws Exception {
    ATSHistoryLoggingService historyLoggingService;
    historyLoggingService = ReflectionUtils.createClazzInstance(ATSHistoryLoggingService.class.getName());
    AppContext appContext = mock(AppContext.class);
    when(appContext.getApplicationID()).thenReturn(ApplicationId.newInstance(0, 1));
    historyLoggingService.setAppContext(appContext);
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    String viewAcls = "nobody nobody_group";
    tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
    tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName());
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
    historyLoggingService.init(tezConf);
    historyLoggingService.start();
    ApplicationId appId = ApplicationId.newInstance(100l, 1);
    TezDAGID tezDAGID = TezDAGID.getInstance(appId, 11);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    DAGPlan dagPlan = DAGPlan.newBuilder().setName("DAGPlanMock").build();
    DAGSubmittedEvent submittedEvent = new DAGSubmittedEvent(tezDAGID, 1, dagPlan, appAttemptId, null, "usr", tezConf, null, null);
    submittedEvent.setHistoryLoggingEnabled(true);
    DAGHistoryEvent event = new DAGHistoryEvent(tezDAGID, submittedEvent);
    historyLoggingService.handle(new DAGHistoryEvent(tezDAGID, submittedEvent));
    Thread.sleep(1000l);
    String url = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_DAG_ID/" + event.getDagID();
    Client client = new Client();
    WebResource resource = client.resource(url);
    ClientResponse response = resource.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(200, response.getStatus());
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    TimelineEntity entity = response.getEntity(TimelineEntity.class);
    assertEquals(entity.getEntityType(), "TEZ_DAG_ID");
    assertEquals(entity.getEvents().get(0).getEventType(), HistoryEventType.DAG_SUBMITTED.toString());
}