Examples with AppContext org.apache.tez.dag.app.AppContext used on opensource projects

Example 46 with AppContext

use of org.apache.tez.dag.app.AppContext in project tez by apache.

the class TestAMNodeTracker method testMultipleSourcesNodeCountUpdated.

@Test(timeout = 5000)
public void testMultipleSourcesNodeCountUpdated() {
    AppContext appContext = mock(AppContext.class);
    AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext);
    doReturn(amNodeTracker).when(appContext).getNodeTracker();
    amNodeTracker.init(new Configuration(false));
    amNodeTracker.start();
    NodeId nodeId1 = NodeId.newInstance("source01", 3333);
    NodeId nodeId2 = NodeId.newInstance("source02", 3333);
    amNodeTracker.nodeSeen(nodeId1, 0);
    amNodeTracker.nodeSeen(nodeId2, 1);
    amNodeTracker.handle(new AMNodeEventNodeCountUpdated(10, 0));
    amNodeTracker.handle(new AMNodeEventNodeCountUpdated(20, 1));
    // NodeCountUpdate does not reflect in getNumNodes.
    assertEquals(1, amNodeTracker.getNumNodes(0));
    assertEquals(1, amNodeTracker.getNumNodes(1));
    assertNotNull(amNodeTracker.get(nodeId1, 0));
    assertNull(amNodeTracker.get(nodeId2, 0));
    assertNull(amNodeTracker.get(nodeId1, 1));
    assertNotNull(amNodeTracker.get(nodeId2, 1));
}

Example 47 with AppContext

use of org.apache.tez.dag.app.AppContext in project tez by apache.

the class TestAMNodeTracker method _testNodeUnhealthyRescheduleTasks.

private void _testNodeUnhealthyRescheduleTasks(boolean rescheduleTasks) {
    AppContext appContext = mock(AppContext.class);
    Configuration conf = new Configuration(false);
    conf.setBoolean(TezConfiguration.TEZ_AM_NODE_UNHEALTHY_RESCHEDULE_TASKS, rescheduleTasks);
    TestEventHandler handler = new TestEventHandler();
    AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
    doReturn(amNodeTracker).when(appContext).getNodeTracker();
    amNodeTracker.init(conf);
    amNodeTracker.start();
    // add a node
    amNodeTracker.handle(new AMNodeEventNodeCountUpdated(1, 0));
    NodeId nodeId = NodeId.newInstance("host1", 1234);
    amNodeTracker.nodeSeen(nodeId, 0);
    AMNodeImpl node = (AMNodeImpl) amNodeTracker.get(nodeId, 0);
    // simulate task starting on node
    ContainerId cid = mock(ContainerId.class);
    amNodeTracker.handle(new AMNodeEventContainerAllocated(nodeId, 0, cid));
    // mark node unhealthy
    NodeReport nodeReport = generateNodeReport(nodeId, NodeState.UNHEALTHY);
    amNodeTracker.handle(new AMNodeEventStateChanged(nodeReport, 0));
    assertEquals(AMNodeState.UNHEALTHY, node.getState());
    // check for task rescheduling events
    if (rescheduleTasks) {
        assertEquals(1, handler.events.size());
        assertEquals(AMContainerEventType.C_NODE_FAILED, handler.events.get(0).getType());
    } else {
        assertEquals(0, handler.events.size());
    }
    amNodeTracker.stop();
}

Example 48 with AppContext

use of org.apache.tez.dag.app.AppContext in project tez by apache.

the class TestRecoveryService method setup.

private void setup(boolean useMockFs, String[][] configs) throws Exception {
    conf = new Configuration();
    if (configs != null) {
        for (String[] config : configs) {
            conf.set(config[0], config[1]);
        }
    }
    appContext = mock(AppContext.class);
    when(appContext.getClock()).thenReturn(new SystemClock());
    when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim());
    when(appContext.getApplicationID()).thenReturn(appId);
    if (useMockFs) {
        fs = mock(FileSystem.class);
        when(appContext.getCurrentRecoveryDir()).thenReturn(new Path("mockfs:///"));
        conf.set("fs.mockfs.impl", MockFileSystem.class.getName());
        MockFileSystem.delegate = fs;
        dagFos = spy(new FSDataOutputStream(new OutputStream() {

            @Override
            public void write(int b) throws IOException {
            }
        }, null));
        summaryFos = spy(new FSDataOutputStream(new OutputStream() {

            @Override
            public void write(int b) throws IOException {
            }
        }, null));
    } else {
        when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(TEST_ROOT_DIR));
        fs = FileSystem.getLocal(conf);
        fs.delete(new Path(TEST_ROOT_DIR), true);
    }
    recoveryService = new MockRecoveryService(appContext);
    conf.setBoolean(RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true);
    recoveryService.init(conf);
    summaryPath = TezCommonUtils.getSummaryRecoveryPath(recoveryService.recoveryPath);
    dagRecoveryPath = TezCommonUtils.getDAGRecoveryPath(recoveryService.recoveryPath, dagId.toString());
    if (useMockFs) {
        when(fs.create(eq(dagRecoveryPath), eq(false), anyInt())).thenReturn(dagFos);
        when(fs.create(eq(summaryPath), eq(false), anyInt())).thenReturn(summaryFos);
    }
}

Example 49 with AppContext

use of org.apache.tez.dag.app.AppContext in project tez by apache.

the class TestATSV15HistoryLoggingService method createService.

private ATSV15HistoryLoggingService createService(int numDagsPerGroup) throws IOException, YarnException {
    ATSV15HistoryLoggingService service = new ATSV15HistoryLoggingService();
    appContext = mock(AppContext.class);
    when(appContext.getApplicationID()).thenReturn(appId);
    when(appContext.getHadoopShim()).thenReturn(new HadoopShim() {
    });
    service.setAppContext(appContext);
    Configuration conf = new Configuration(false);
    if (numDagsPerGroup != -1) {
        conf.setInt(TezConfiguration.TEZ_HISTORY_LOGGING_TIMELINE_NUM_DAGS_PER_GROUP, numDagsPerGroup);
    }
    service.init(conf);
    // Set timeline service.
    timelineClient = mock(TimelineClient.class);
    entityLog = new HashMap<>();
    // timelineClient.init(conf);
    when(timelineClient.getDelegationToken(anyString())).thenReturn(null);
    when(timelineClient.renewDelegationToken(Matchers.<Token<TimelineDelegationTokenIdentifier>>any())).thenReturn(0L);
    when(timelineClient.putEntities(Matchers.<TimelineEntity>anyVararg())).thenAnswer(new Answer() {

        @Override
        public TimelinePutResponse answer(InvocationOnMock invocation) throws Throwable {
            return putEntityHelper(DEFAULT_GROUP_ID, invocation.getArguments(), 0);
        }
    });
    when(timelineClient.putEntities(any(ApplicationAttemptId.class), any(TimelineEntityGroupId.class), Matchers.<TimelineEntity>anyVararg())).thenAnswer(new Answer() {

        @Override
        public TimelinePutResponse answer(InvocationOnMock invocation) throws Throwable {
            return putEntityHelper(invocation.getArgumentAt(1, TimelineEntityGroupId.class), invocation.getArguments(), 2);
        }
    });
    service.timelineClient = timelineClient;
    return service;
}

Example 50 with AppContext

use of org.apache.tez.dag.app.AppContext in project tez by apache.

the class TestATSHistoryWithACLs method testDagLoggingEnabled.

/**
 * use mini cluster to verify data do push to ats when
 * the dag logging flag in dagsubmitted event is set on
 * @throws Exception
 */
@Test(timeout = 50000)
public void testDagLoggingEnabled() throws Exception {
    ATSHistoryLoggingService historyLoggingService;
    historyLoggingService = ReflectionUtils.createClazzInstance(ATSHistoryLoggingService.class.getName());
    AppContext appContext = mock(AppContext.class);
    when(appContext.getApplicationID()).thenReturn(ApplicationId.newInstance(0, 1));
    historyLoggingService.setAppContext(appContext);
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    String viewAcls = "nobody nobody_group";
    tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls);
    tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName());
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
    historyLoggingService.init(tezConf);
    historyLoggingService.start();
    ApplicationId appId = ApplicationId.newInstance(100l, 1);
    TezDAGID tezDAGID = TezDAGID.getInstance(appId, 11);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    DAGPlan dagPlan = DAGPlan.newBuilder().setName("DAGPlanMock").build();
    DAGSubmittedEvent submittedEvent = new DAGSubmittedEvent(tezDAGID, 1, dagPlan, appAttemptId, null, "usr", tezConf, null, null);
    submittedEvent.setHistoryLoggingEnabled(true);
    DAGHistoryEvent event = new DAGHistoryEvent(tezDAGID, submittedEvent);
    historyLoggingService.handle(new DAGHistoryEvent(tezDAGID, submittedEvent));
    Thread.sleep(1000l);
    String url = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_DAG_ID/" + event.getDagID();
    Client client = new Client();
    WebResource resource = client.resource(url);
    ClientResponse response = resource.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(200, response.getStatus());
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    TimelineEntity entity = response.getEntity(TimelineEntity.class);
    assertEquals(entity.getEntityType(), "TEZ_DAG_ID");
    assertEquals(entity.getEvents().get(0).getEventType(), HistoryEventType.DAG_SUBMITTED.toString());
}