Examples with AppContext org.apache.tez.dag.app.AppContext used on opensource projects

Example 21 with AppContext

use of org.apache.tez.dag.app.AppContext in project tez by apache.

the class TestAMNodeTracker method testNodeCompletedAndCleanup.

@Test(timeout = 10000L)
public void testNodeCompletedAndCleanup() {
    AppContext appContext = mock(AppContext.class);
    Configuration conf = new Configuration(false);
    conf.setInt(TezConfiguration.TEZ_AM_MAX_TASK_FAILURES_PER_NODE, 2);
    TestEventHandler handler = new TestEventHandler();
    AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
    doReturn(amNodeTracker).when(appContext).getNodeTracker();
    AMContainerMap amContainerMap = mock(AMContainerMap.class);
    TaskSchedulerManager taskSchedulerManager = mock(TaskSchedulerManager.class);
    dispatcher.register(AMNodeEventType.class, amNodeTracker);
    dispatcher.register(AMContainerEventType.class, amContainerMap);
    dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
    amNodeTracker.init(conf);
    amNodeTracker.start();
    try {
        NodeId nodeId = NodeId.newInstance("fakenode", 3333);
        amNodeTracker.nodeSeen(nodeId, 0);
        AMNode amNode = amNodeTracker.get(nodeId, 0);
        ContainerId[] containerIds = new ContainerId[7];
        // Start 5 containers.
        for (int i = 0; i < 5; i++) {
            containerIds[i] = mock(ContainerId.class);
            amNodeTracker.handle(new AMNodeEventContainerAllocated(nodeId, 0, containerIds[i]));
        }
        assertEquals(5, amNode.getContainers().size());
        // Finnish 1st dag
        amNodeTracker.dagComplete(mock(DAG.class));
        assertEquals(5, amNode.getContainers().size());
        // Mark 2 as complete. Finish 2nd dag.
        for (int i = 0; i < 2; i++) {
            amNodeTracker.handle(new AMNodeEventContainerCompleted(nodeId, 0, containerIds[i]));
        }
        amNodeTracker.dagComplete(mock(DAG.class));
        assertEquals(3, amNode.getContainers().size());
        // Add 2 more containers. Mark all as complete. Finish 3rd dag.
        for (int i = 5; i < 7; i++) {
            containerIds[i] = mock(ContainerId.class);
            amNodeTracker.handle(new AMNodeEventContainerAllocated(nodeId, 0, containerIds[i]));
        }
        assertEquals(5, amNode.getContainers().size());
        amNodeTracker.dagComplete(mock(DAG.class));
        assertEquals(5, amNode.getContainers().size());
        amNodeTracker.dagComplete(mock(DAG.class));
        assertEquals(5, amNode.getContainers().size());
        for (int i = 2; i < 7; i++) {
            amNodeTracker.handle(new AMNodeEventContainerCompleted(nodeId, 0, containerIds[i]));
        }
        assertEquals(5, amNode.getContainers().size());
        amNodeTracker.dagComplete(mock(DAG.class));
        assertEquals(0, amNode.getContainers().size());
    } finally {
        amNodeTracker.stop();
    }
}

Example 22 with AppContext

use of org.apache.tez.dag.app.AppContext in project tez by apache.

the class TestHistoryEventHandler method createHandler.

private HistoryEventHandler createHandler(HistoryLogLevel logLevel) {
    Configuration conf = new Configuration(baseConfig);
    conf.setBoolean(TezConfiguration.DAG_RECOVERY_ENABLED, false);
    conf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, InMemoryHistoryLoggingService.class.getName());
    if (logLevel != null) {
        conf.setEnum(TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL, logLevel);
    }
    DAG dag = mock(DAG.class);
    when(dag.getConf()).thenReturn(conf);
    AppContext appContext = mock(AppContext.class);
    when(appContext.getApplicationID()).thenReturn(appId);
    when(appContext.getHadoopShim()).thenReturn(new HadoopShim() {
    });
    when(appContext.getAMConf()).thenReturn(conf);
    when(appContext.getCurrentDAG()).thenReturn(dag);
    HistoryEventHandler handler = new HistoryEventHandler(appContext);
    handler.init(conf);
    return handler;
}

Example 23 with AppContext

use of org.apache.tez.dag.app.AppContext in project tez by apache.

the class TestATSHistoryLoggingService method setup.

@Before
public void setup() throws Exception {
    appContext = mock(AppContext.class);
    historyACLPolicyManager = mock(HistoryACLPolicyManager.class);
    atsHistoryLoggingService = new ATSHistoryLoggingService();
    atsHistoryLoggingService.setAppContext(appContext);
    conf = new Configuration(false);
    conf.setLong(TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS, 1000l);
    conf.setInt(TezConfiguration.YARN_ATS_MAX_EVENTS_PER_BATCH, 2);
    conf.setBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS, true);
    conf.set(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID, "test-domain");
    atsInvokeCounter = 0;
    atsEntitiesCounter = 0;
    atsHistoryLoggingService.init(conf);
    atsHistoryLoggingService.historyACLPolicyManager = historyACLPolicyManager;
    atsHistoryLoggingService.timelineClient = mock(TimelineClient.class);
    when(appContext.getClock()).thenReturn(clock);
    when(appContext.getCurrentDAGID()).thenReturn(null);
    when(appContext.getApplicationID()).thenReturn(appId);
    when(atsHistoryLoggingService.timelineClient.putEntities(Matchers.<TimelineEntity[]>anyVararg())).thenAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            ++atsInvokeCounter;
            atsEntitiesCounter += invocation.getArguments().length;
            try {
                Thread.sleep(500l);
            } catch (InterruptedException e) {
            // do nothing
            }
            return null;
        }
    });
}

Example 24 with AppContext

use of org.apache.tez.dag.app.AppContext in project tez by apache.

the class TestATSHistoryWithACLs method testDagLoggingDisabled.

/**
 * use mini cluster to verify data do not push to ats when the daglogging flag
 * in dagsubmittedevent is set off
 * @throws Exception
 */
@Test(timeout = 50000)
public void testDagLoggingDisabled() throws Exception {
    ATSHistoryLoggingService historyLoggingService;
    historyLoggingService = ReflectionUtils.createClazzInstance(ATSHistoryLoggingService.class.getName());
    AppContext appContext = mock(AppContext.class);
    when(appContext.getApplicationID()).thenReturn(ApplicationId.newInstance(0, 1));
    historyLoggingService.setAppContext(appContext);
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    String viewAcls = "nobody nobody_group";
    tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
    tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName());
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
    historyLoggingService.init(tezConf);
    historyLoggingService.start();
    ApplicationId appId = ApplicationId.newInstance(100l, 1);
    TezDAGID tezDAGID = TezDAGID.getInstance(appId, 100);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    DAGPlan dagPlan = DAGPlan.newBuilder().setName("DAGPlanMock").build();
    DAGSubmittedEvent submittedEvent = new DAGSubmittedEvent(tezDAGID, 1, dagPlan, appAttemptId, null, "usr", tezConf, null, null);
    submittedEvent.setHistoryLoggingEnabled(false);
    DAGHistoryEvent event = new DAGHistoryEvent(tezDAGID, submittedEvent);
    historyLoggingService.handle(new DAGHistoryEvent(tezDAGID, submittedEvent));
    Thread.sleep(1000l);
    String url = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_DAG_ID/" + event.getDagID();
    Client client = new Client();
    WebResource resource = client.resource(url);
    ClientResponse response = resource.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(404, response.getStatus());
}

Example 25 with AppContext

use of org.apache.tez.dag.app.AppContext in project tez by apache.

the class TestDAGClientHandler method testDAGClientHandler.

@Test(timeout = 5000)
public void testDAGClientHandler() throws TezException {
    TezDAGID mockTezDAGId = mock(TezDAGID.class);
    when(mockTezDAGId.getId()).thenReturn(1);
    when(mockTezDAGId.toString()).thenReturn("dag_9999_0001_1");
    DAG mockDAG = mock(DAG.class);
    when(mockDAG.getID()).thenReturn(mockTezDAGId);
    DAGStatusBuilder mockDagStatusBuilder = mock(DAGStatusBuilder.class);
    when(mockDAG.getDAGStatus(anySetOf(StatusGetOpts.class))).thenReturn(mockDagStatusBuilder);
    VertexStatusBuilder mockVertexStatusBuilder = mock(VertexStatusBuilder.class);
    when(mockDAG.getVertexStatus(anyString(), anySetOf(StatusGetOpts.class))).thenReturn(mockVertexStatusBuilder);
    DAGAppMaster mockDagAM = mock(DAGAppMaster.class);
    when(mockDagAM.getState()).thenReturn(DAGAppMasterState.RUNNING);
    AppContext mockAppContext = mock(AppContext.class);
    when(mockDagAM.getContext()).thenReturn(mockAppContext);
    when(mockDagAM.getContext().getCurrentDAG()).thenReturn(mockDAG);
    when(mockAppContext.getClock()).thenReturn(new SystemClock());
    DAGClientHandler dagClientHandler = new DAGClientHandler(mockDagAM);
    // getAllDAGs()
    assertEquals(1, dagClientHandler.getAllDAGs().size());
    assertEquals("dag_9999_0001_1", dagClientHandler.getAllDAGs().get(0));
    // getDAGStatus
    try {
        dagClientHandler.getDAGStatus("dag_9999_0001_2", Sets.newSet(StatusGetOpts.GET_COUNTERS));
        fail("should not come here");
    } catch (TezException e) {
        assertTrue(e.getMessage().contains("Unknown dagId"));
    }
    DAGStatus dagStatus = dagClientHandler.getDAGStatus("dag_9999_0001_1", Sets.newSet(StatusGetOpts.GET_COUNTERS));
    assertEquals(mockDagStatusBuilder, dagStatus);
    // getVertexStatus
    try {
        dagClientHandler.getVertexStatus("dag_9999_0001_2", "v1", Sets.newSet(StatusGetOpts.GET_COUNTERS));
        fail("should not come here");
    } catch (TezException e) {
        assertTrue(e.getMessage().contains("Unknown dagId"));
    }
    VertexStatus vertexStatus = dagClientHandler.getVertexStatus("dag_9999_0001_1", "v1", Sets.newSet(StatusGetOpts.GET_COUNTERS));
    assertEquals(mockVertexStatusBuilder, vertexStatus);
    // getTezAppMasterStatus
    when(mockDagAM.isSession()).thenReturn(false);
    assertEquals(TezAppMasterStatus.RUNNING, dagClientHandler.getTezAppMasterStatus());
    when(mockDagAM.isSession()).thenReturn(true);
    when(mockDagAM.getState()).thenReturn(DAGAppMasterState.INITED);
    assertEquals(TezAppMasterStatus.INITIALIZING, dagClientHandler.getTezAppMasterStatus());
    when(mockDagAM.getState()).thenReturn(DAGAppMasterState.ERROR);
    assertEquals(TezAppMasterStatus.SHUTDOWN, dagClientHandler.getTezAppMasterStatus());
    // tryKillDAG
    try {
        dagClientHandler.tryKillDAG("dag_9999_0001_2");
        fail("should not come here");
    } catch (TezException e) {
        assertTrue(e.getMessage().contains("Unknown dagId"));
    }
    dagClientHandler.tryKillDAG("dag_9999_0001_1");
    ArgumentCaptor<DAG> eventCaptor = ArgumentCaptor.forClass(DAG.class);
    verify(mockDagAM, times(1)).tryKillDAG(eventCaptor.capture(), contains("Sending client kill from"));
    assertEquals(1, eventCaptor.getAllValues().size());
    assertTrue(eventCaptor.getAllValues().get(0) instanceof DAG);
    assertEquals("dag_9999_0001_1", ((DAG) eventCaptor.getAllValues().get(0)).getID().toString());
    // submitDAG
    DAGPlan dagPlan = DAGPlan.getDefaultInstance();
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    dagClientHandler.submitDAG(dagPlan, localResources);
    verify(mockDagAM).submitDAGToAppMaster(dagPlan, localResources);
    // shutdown
    dagClientHandler.shutdownAM();
    verify(mockDagAM).shutdownTezAM(contains("Received message to shutdown AM from"));
}