Examples with Application org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application used on opensource projects

Example 41 with Application

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application in project hadoop by apache.

the class TestResourceLocalizationService method testLocalizationHeartbeat.

@Test(timeout = 10000)
// mocked generics
@SuppressWarnings("unchecked")
public void testLocalizationHeartbeat() throws Exception {
    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[1];
    // Making sure that we have only one local disk so that it will only be
    // selected for consecutive resource localization calls.  This is required
    // to test LocalCacheDirectoryManager.
    localDirs.add(lfs.makeQualified(new Path(basedir, 0 + "")));
    sDirs[0] = localDirs.get(0).toString();
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    // Adding configuration to make sure there is only one file per
    // directory
    conf.set(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY, "37");
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    ContainerExecutor exec = mock(ContainerExecutor.class);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    DeletionService delServiceReal = new DeletionService(exec);
    DeletionService delService = spy(delServiceReal);
    delService.init(new Configuration());
    delService.start();
    ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, nmContext);
    ResourceLocalizationService spyService = spy(rawService);
    doReturn(mockServer).when(spyService).createServer();
    doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
    FsPermission defaultPermission = FsPermission.getDirDefault().applyUMask(lfs.getUMask());
    FsPermission nmPermission = ResourceLocalizationService.NM_PRIVATE_PERM.applyUMask(lfs.getUMask());
    final Path userDir = new Path(sDirs[0].substring("file:".length()), ContainerLocalizer.USERCACHE);
    final Path fileDir = new Path(sDirs[0].substring("file:".length()), ContainerLocalizer.FILECACHE);
    final Path sysDir = new Path(sDirs[0].substring("file:".length()), ResourceLocalizationService.NM_PRIVATE_DIR);
    final FileStatus fs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, defaultPermission, "", "", new Path(sDirs[0]));
    final FileStatus nmFs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, nmPermission, "", "", sysDir);
    doAnswer(new Answer<FileStatus>() {

        @Override
        public FileStatus answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            if (args.length > 0) {
                if (args[0].equals(userDir) || args[0].equals(fileDir)) {
                    return fs;
                }
            }
            return nmFs;
        }
    }).when(spylfs).getFileStatus(isA(Path.class));
    try {
        spyService.init(conf);
        spyService.start();
        // init application
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
        when(app.getUser()).thenReturn("user0");
        when(app.getAppId()).thenReturn(appId);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        ArgumentMatcher<ApplicationEvent> matchesAppInit = new ArgumentMatcher<ApplicationEvent>() {

            @Override
            public boolean matches(Object o) {
                ApplicationEvent evt = (ApplicationEvent) o;
                return evt.getType() == ApplicationEventType.APPLICATION_INITED && appId == evt.getApplicationID();
            }
        };
        dispatcher.await();
        verify(applicationBus).handle(argThat(matchesAppInit));
        // init container rsrc, localizer
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        final Container c = getMockContainer(appId, 42, "user0");
        FSDataOutputStream out = new FSDataOutputStream(new DataOutputBuffer(), null);
        doReturn(out).when(spylfs).createInternal(isA(Path.class), isA(EnumSet.class), isA(FsPermission.class), anyInt(), anyShort(), anyLong(), isA(Progressable.class), isA(ChecksumOpt.class), anyBoolean());
        final LocalResource resource1 = getPrivateMockedResource(r);
        LocalResource resource2 = null;
        do {
            resource2 = getPrivateMockedResource(r);
        } while (resource2 == null || resource2.equals(resource1));
        LocalResource resource3 = null;
        do {
            resource3 = getPrivateMockedResource(r);
        } while (resource3 == null || resource3.equals(resource1) || resource3.equals(resource2));
        // above call to make sure we don't get identical resources.
        final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
        final LocalResourceRequest req2 = new LocalResourceRequest(resource2);
        final LocalResourceRequest req3 = new LocalResourceRequest(resource3);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        List<LocalResourceRequest> privateResourceList = new ArrayList<LocalResourceRequest>();
        privateResourceList.add(req1);
        privateResourceList.add(req2);
        privateResourceList.add(req3);
        rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList);
        spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs));
        // Sigh. Thread init of private localizer not accessible
        Thread.sleep(1000);
        dispatcher.await();
        String appStr = appId.toString();
        String ctnrStr = c.getContainerId().toString();
        ArgumentCaptor<LocalizerStartContext> contextCaptor = ArgumentCaptor.forClass(LocalizerStartContext.class);
        verify(exec).startLocalizer(contextCaptor.capture());
        LocalizerStartContext context = contextCaptor.getValue();
        Path localizationTokenPath = context.getNmPrivateContainerTokens();
        assertEquals("user0", context.getUser());
        assertEquals(appStr, context.getAppId());
        assertEquals(ctnrStr, context.getLocId());
        // heartbeat from localizer
        LocalResourceStatus rsrc1success = mock(LocalResourceStatus.class);
        LocalResourceStatus rsrc2pending = mock(LocalResourceStatus.class);
        LocalResourceStatus rsrc2success = mock(LocalResourceStatus.class);
        LocalResourceStatus rsrc3success = mock(LocalResourceStatus.class);
        LocalizerStatus stat = mock(LocalizerStatus.class);
        when(stat.getLocalizerId()).thenReturn(ctnrStr);
        when(rsrc1success.getResource()).thenReturn(resource1);
        when(rsrc2pending.getResource()).thenReturn(resource2);
        when(rsrc2success.getResource()).thenReturn(resource2);
        when(rsrc3success.getResource()).thenReturn(resource3);
        when(rsrc1success.getLocalSize()).thenReturn(4344L);
        when(rsrc2success.getLocalSize()).thenReturn(2342L);
        when(rsrc3success.getLocalSize()).thenReturn(5345L);
        URL locPath = getPath("/cache/private/blah");
        when(rsrc1success.getLocalPath()).thenReturn(locPath);
        when(rsrc2success.getLocalPath()).thenReturn(locPath);
        when(rsrc3success.getLocalPath()).thenReturn(locPath);
        when(rsrc1success.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS);
        when(rsrc2pending.getStatus()).thenReturn(ResourceStatusType.FETCH_PENDING);
        when(rsrc2success.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS);
        when(rsrc3success.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS);
        // Four heartbeats with sending:
        // 1 - empty
        // 2 - resource1 FETCH_SUCCESS
        // 3 - resource2 FETCH_PENDING
        // 4 - resource2 FETCH_SUCCESS, resource3 FETCH_SUCCESS
        List<LocalResourceStatus> rsrcs4 = new ArrayList<LocalResourceStatus>();
        rsrcs4.add(rsrc2success);
        rsrcs4.add(rsrc3success);
        when(stat.getResources()).thenReturn(Collections.<LocalResourceStatus>emptyList()).thenReturn(Collections.singletonList(rsrc1success)).thenReturn(Collections.singletonList(rsrc2pending)).thenReturn(rsrcs4).thenReturn(Collections.<LocalResourceStatus>emptyList());
        String localPath = Path.SEPARATOR + ContainerLocalizer.USERCACHE + Path.SEPARATOR + "user0" + Path.SEPARATOR + ContainerLocalizer.FILECACHE;
        // First heartbeat
        LocalizerHeartbeatResponse response = spyService.heartbeat(stat);
        assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
        assertEquals(1, response.getResourceSpecs().size());
        assertEquals(req1, new LocalResourceRequest(response.getResourceSpecs().get(0).getResource()));
        URL localizedPath = response.getResourceSpecs().get(0).getDestinationDirectory();
        // Appending to local path unique number(10) generated as a part of
        // LocalResourcesTracker
        assertTrue(localizedPath.getFile().endsWith(localPath + Path.SEPARATOR + "10"));
        // Second heartbeat
        response = spyService.heartbeat(stat);
        assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
        assertEquals(1, response.getResourceSpecs().size());
        assertEquals(req2, new LocalResourceRequest(response.getResourceSpecs().get(0).getResource()));
        localizedPath = response.getResourceSpecs().get(0).getDestinationDirectory();
        // Resource's destination path should be now inside sub directory 0 as
        // LocalCacheDirectoryManager will be used and we have restricted number
        // of files per directory to 1.
        assertTrue(localizedPath.getFile().endsWith(localPath + Path.SEPARATOR + "0" + Path.SEPARATOR + "11"));
        // Third heartbeat
        response = spyService.heartbeat(stat);
        assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
        assertEquals(1, response.getResourceSpecs().size());
        assertEquals(req3, new LocalResourceRequest(response.getResourceSpecs().get(0).getResource()));
        localizedPath = response.getResourceSpecs().get(0).getDestinationDirectory();
        assertTrue(localizedPath.getFile().endsWith(localPath + Path.SEPARATOR + "1" + Path.SEPARATOR + "12"));
        response = spyService.heartbeat(stat);
        assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
        spyService.handle(new ContainerLocalizationEvent(LocalizationEventType.CONTAINER_RESOURCES_LOCALIZED, c));
        // get shutdown after receive CONTAINER_RESOURCES_LOCALIZED event
        response = spyService.heartbeat(stat);
        assertEquals(LocalizerAction.DIE, response.getLocalizerAction());
        dispatcher.await();
        // verify container notification
        ArgumentMatcher<ContainerEvent> matchesContainerLoc = new ArgumentMatcher<ContainerEvent>() {

            @Override
            public boolean matches(Object o) {
                ContainerEvent evt = (ContainerEvent) o;
                return evt.getType() == ContainerEventType.RESOURCE_LOCALIZED && c.getContainerId() == evt.getContainerID();
            }
        };
        // total 3 resource localzation calls. one for each resource.
        verify(containerBus, times(3)).handle(argThat(matchesContainerLoc));
        // Verify deletion of localization token.
        verify(delService).delete((String) isNull(), eq(localizationTokenPath));
    } finally {
        spyService.stop();
        dispatcher.stop();
        delService.stop();
    }
}

Example 42 with Application

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application in project hadoop by apache.

the class TestLogAggregationService method testLogAggregationServiceWithPatternsAndIntervals.

@SuppressWarnings("resource")
@Test(timeout = 50000)
public void testLogAggregationServiceWithPatternsAndIntervals() throws Exception {
    LogAggregationContext logAggregationContext = Records.newRecord(LogAggregationContext.class);
    // set IncludePattern and RolledLogsIncludePattern.
    // When the app is running, we only aggregate the log with
    // the name stdout. After the app finishes, we only aggregate
    // the log with the name std_final.
    logAggregationContext.setRolledLogsIncludePattern("stdout");
    logAggregationContext.setIncludePattern("std_final");
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    //configure YarnConfiguration.NM_REMOTE_APP_LOG_DIR to
    //have fully qualified path
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.toURI().toString());
    this.conf.setLong(YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, 3600);
    this.conf.setLong(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 3600);
    ApplicationId application = BuilderUtils.newApplicationId(System.currentTimeMillis(), 1);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(application, 1);
    ContainerId container = createContainer(appAttemptId, 1, ContainerType.APPLICATION_MASTER);
    ConcurrentMap<ApplicationId, Application> maps = this.context.getApplications();
    Application app = mock(Application.class);
    maps.put(application, app);
    when(app.getContainers()).thenReturn(this.context.getContainers());
    LogAggregationService logAggregationService = new LogAggregationService(dispatcher, context, this.delSrvc, super.dirsHandler);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    // AppLogDir should be created
    File appLogDir = new File(localLogDir, ConverterUtils.toString(application));
    appLogDir.mkdir();
    logAggregationService.handle(new LogHandlerAppStartedEvent(application, this.user, null, this.acls, logAggregationContext));
    // Simulate log-file creation
    // create std_final in log directory which will not be aggregated
    // until the app finishes.
    String[] logFilesWithFinalLog = new String[] { "stdout", "std_final" };
    writeContainerLogs(appLogDir, container, logFilesWithFinalLog);
    // Do log aggregation
    AppLogAggregatorImpl aggregator = (AppLogAggregatorImpl) logAggregationService.getAppLogAggregators().get(application);
    aggregator.doLogAggregationOutOfBand();
    Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 1, false, null));
    String[] logFiles = new String[] { "stdout" };
    verifyContainerLogs(logAggregationService, application, new ContainerId[] { container }, logFiles, 1, true);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container, 0));
    dispatcher.await();
    // Do the log aggregation after ContainerFinishedEvent but before
    // AppFinishedEvent. The std_final is expected to be aggregated this time
    // even if the app is running but the container finishes.
    aggregator.doLogAggregationOutOfBand();
    Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 2, false, null));
    // This container finishes.
    // The log "std_final" should be aggregated this time.
    String[] logFinalLog = new String[] { "std_final" };
    verifyContainerLogs(logAggregationService, application, new ContainerId[] { container }, logFinalLog, 1, true);
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application));
    logAggregationService.stop();
}

Example 43 with Application

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application in project hadoop by apache.

the class TestResourceLocalizationService method testResourceRelease.

@Test
// mocked generics
@SuppressWarnings("unchecked")
public void testResourceRelease() throws Exception {
    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[4];
    for (int i = 0; i < 4; ++i) {
        localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
        sDirs[i] = localDirs.get(i).toString();
    }
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    LocalizerTracker mockLocallilzerTracker = mock(LocalizerTracker.class);
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    //Ignore actual localization
    EventHandler<LocalizerEvent> localizerBus = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerBus);
    ContainerExecutor exec = mock(ContainerExecutor.class);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    DeletionService delService = new DeletionService(exec);
    delService.init(new Configuration());
    delService.start();
    ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, nmContext);
    ResourceLocalizationService spyService = spy(rawService);
    doReturn(mockServer).when(spyService).createServer();
    doReturn(mockLocallilzerTracker).when(spyService).createLocalizerTracker(isA(Configuration.class));
    doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
    try {
        spyService.init(conf);
        spyService.start();
        final String user = "user0";
        // init application
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
        when(app.getUser()).thenReturn(user);
        when(app.getAppId()).thenReturn(appId);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        dispatcher.await();
        //Get a handle on the trackers after they're setup with INIT_APP_RESOURCES
        LocalResourcesTracker appTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user, appId);
        LocalResourcesTracker privTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user, appId);
        LocalResourcesTracker pubTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, appId);
        // init container.
        final Container c = getMockContainer(appId, 42, user);
        // init resources
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        // Send localization requests for one resource of each type.
        final LocalResource privResource = getPrivateMockedResource(r);
        final LocalResourceRequest privReq = new LocalResourceRequest(privResource);
        final LocalResource pubResource = getPublicMockedResource(r);
        final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource);
        final LocalResource pubResource2 = getPublicMockedResource(r);
        final LocalResourceRequest pubReq2 = new LocalResourceRequest(pubResource2);
        final LocalResource appResource = getAppMockedResource(r);
        final LocalResourceRequest appReq = new LocalResourceRequest(appResource);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req.put(LocalResourceVisibility.PRIVATE, Collections.singletonList(privReq));
        req.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq));
        req.put(LocalResourceVisibility.APPLICATION, Collections.singletonList(appReq));
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req2 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req2.put(LocalResourceVisibility.PRIVATE, Collections.singletonList(privReq));
        req2.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq2));
        Set<LocalResourceRequest> pubRsrcs = new HashSet<LocalResourceRequest>();
        pubRsrcs.add(pubReq);
        pubRsrcs.add(pubReq2);
        // Send Request event
        spyService.handle(new ContainerLocalizationRequestEvent(c, req));
        spyService.handle(new ContainerLocalizationRequestEvent(c, req2));
        dispatcher.await();
        int privRsrcCount = 0;
        for (LocalizedResource lr : privTracker) {
            privRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 2, lr.getRefCount());
            Assert.assertEquals(privReq, lr.getRequest());
        }
        Assert.assertEquals(1, privRsrcCount);
        int pubRsrcCount = 0;
        for (LocalizedResource lr : pubTracker) {
            pubRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount());
            pubRsrcs.remove(lr.getRequest());
        }
        Assert.assertEquals(0, pubRsrcs.size());
        Assert.assertEquals(2, pubRsrcCount);
        int appRsrcCount = 0;
        for (LocalizedResource lr : appTracker) {
            appRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount());
            Assert.assertEquals(appReq, lr.getRequest());
        }
        Assert.assertEquals(1, appRsrcCount);
        //Send Cleanup Event
        spyService.handle(new ContainerLocalizationCleanupEvent(c, req));
        verify(mockLocallilzerTracker).cleanupPrivLocalizers("container_314159265358979_0003_01_000042");
        req2.remove(LocalResourceVisibility.PRIVATE);
        spyService.handle(new ContainerLocalizationCleanupEvent(c, req2));
        dispatcher.await();
        pubRsrcs.add(pubReq);
        pubRsrcs.add(pubReq2);
        privRsrcCount = 0;
        for (LocalizedResource lr : privTracker) {
            privRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount());
            Assert.assertEquals(privReq, lr.getRequest());
        }
        Assert.assertEquals(1, privRsrcCount);
        pubRsrcCount = 0;
        for (LocalizedResource lr : pubTracker) {
            pubRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 0, lr.getRefCount());
            pubRsrcs.remove(lr.getRequest());
        }
        Assert.assertEquals(0, pubRsrcs.size());
        Assert.assertEquals(2, pubRsrcCount);
        appRsrcCount = 0;
        for (LocalizedResource lr : appTracker) {
            appRsrcCount++;
        }
        Assert.assertEquals(0, appRsrcCount);
    } finally {
        dispatcher.stop();
        delService.stop();
    }
}

Example 44 with Application

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application in project hadoop by apache.

the class TestLogAggregationService method testAddNewTokenSentFromRMForLogAggregation.

@Test(timeout = 20000)
public void testAddNewTokenSentFromRMForLogAggregation() throws Exception {
    Configuration conf = new YarnConfiguration();
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    UserGroupInformation.setConfiguration(conf);
    ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1);
    Application mockApp = mock(Application.class);
    when(mockApp.getContainers()).thenReturn(new HashMap<ContainerId, Container>());
    this.context.getApplications().put(application1, mockApp);
    @SuppressWarnings("resource") LogAggregationService logAggregationService = new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    logAggregationService.handle(new LogHandlerAppStartedEvent(application1, this.user, null, this.acls, Records.newRecord(LogAggregationContext.class)));
    // Inject new token for log-aggregation after app log-aggregator init
    Text userText1 = new Text("user1");
    RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier(userText1, new Text("renewer1"), userText1);
    final Token<RMDelegationTokenIdentifier> token1 = new Token<RMDelegationTokenIdentifier>(dtId1.getBytes(), "password1".getBytes(), dtId1.getKind(), new Text("service1"));
    Credentials credentials = new Credentials();
    credentials.addToken(userText1, token1);
    this.context.getSystemCredentialsForApps().put(application1, credentials);
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application1));
    final UserGroupInformation ugi = ((AppLogAggregatorImpl) logAggregationService.getAppLogAggregators().get(application1)).getUgi();
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        public Boolean get() {
            boolean hasNewToken = false;
            for (Token<?> token : ugi.getCredentials().getAllTokens()) {
                if (token.equals(token1)) {
                    hasNewToken = true;
                }
            }
            return hasNewToken;
        }
    }, 1000, 20000);
    logAggregationService.stop();
}

Example 45 with Application

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application in project hadoop by apache.

the class TestLogAggregationService method testLogAggregationService.

@SuppressWarnings("unchecked")
private void testLogAggregationService(boolean retentionSizeLimitation) throws Exception {
    LogAggregationContext logAggregationContextWithInterval = Records.newRecord(LogAggregationContext.class);
    // set IncludePattern/excludePattern in rolling fashion
    // we expect all the logs except std_final will be uploaded
    // when app is running. The std_final will be uploaded when
    // the app finishes.
    logAggregationContextWithInterval.setRolledLogsIncludePattern(".*");
    logAggregationContextWithInterval.setRolledLogsExcludePattern("std_final");
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    //configure YarnConfiguration.NM_REMOTE_APP_LOG_DIR to
    //have fully qualified path
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.toURI().toString());
    this.conf.setLong(YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, 3600);
    if (retentionSizeLimitation) {
        // set the retention size as 1. The number of logs for one application
        // in one NM should be 1.
        this.conf.setInt(YarnConfiguration.NM_PREFIX + "log-aggregation.num-log-files-per-app", 1);
    }
    // by setting this configuration, the log files will not be deleted immediately after
    // they are aggregated to remote directory.
    // We could use it to test whether the previous aggregated log files will be aggregated
    // again in next cycle.
    this.conf.setLong(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 3600);
    ApplicationId application = BuilderUtils.newApplicationId(System.currentTimeMillis(), 1);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(application, 1);
    ContainerId container = createContainer(appAttemptId, 1, ContainerType.APPLICATION_MASTER);
    ConcurrentMap<ApplicationId, Application> maps = this.context.getApplications();
    Application app = mock(Application.class);
    maps.put(application, app);
    when(app.getContainers()).thenReturn(this.context.getContainers());
    LogAggregationService logAggregationService = new LogAggregationService(dispatcher, context, this.delSrvc, super.dirsHandler);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    // AppLogDir should be created
    File appLogDir = new File(localLogDir, application.toString());
    appLogDir.mkdir();
    logAggregationService.handle(new LogHandlerAppStartedEvent(application, this.user, null, this.acls, logAggregationContextWithInterval));
    LogFileStatusInLastCycle logFileStatusInLastCycle = null;
    // Simulate log-file creation
    // create std_final in log directory which will not be aggregated
    // until the app finishes.
    String[] logFiles1WithFinalLog = new String[] { "stdout", "stderr", "syslog", "std_final" };
    String[] logFiles1 = new String[] { "stdout", "stderr", "syslog" };
    writeContainerLogs(appLogDir, container, logFiles1WithFinalLog);
    // Do log aggregation
    AppLogAggregatorImpl aggregator = (AppLogAggregatorImpl) logAggregationService.getAppLogAggregators().get(application);
    aggregator.doLogAggregationOutOfBand();
    if (retentionSizeLimitation) {
        Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 1, true, null));
    } else {
        Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 1, false, null));
    }
    // Container logs should be uploaded
    logFileStatusInLastCycle = verifyContainerLogs(logAggregationService, application, new ContainerId[] { container }, logFiles1, 3, true);
    for (String logFile : logFiles1) {
        Assert.assertTrue(logFileStatusInLastCycle.getLogFileTypesInLastCycle().contains(logFile));
    }
    // Make sure the std_final is not uploaded.
    Assert.assertFalse(logFileStatusInLastCycle.getLogFileTypesInLastCycle().contains("std_final"));
    Thread.sleep(2000);
    // There is no log generated at this time. Do the log aggregation again.
    aggregator.doLogAggregationOutOfBand();
    // Same logs will not be aggregated again.
    // Only one aggregated log file in Remote file directory.
    Assert.assertTrue("Only one aggregated log file in Remote file directory expected", waitAndCheckLogNum(logAggregationService, application, 50, 1, true, null));
    Thread.sleep(2000);
    // Do log aggregation
    String[] logFiles2 = new String[] { "stdout_1", "stderr_1", "syslog_1" };
    writeContainerLogs(appLogDir, container, logFiles2);
    aggregator.doLogAggregationOutOfBand();
    if (retentionSizeLimitation) {
        Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 1, true, logFileStatusInLastCycle.getLogFilePathInLastCycle()));
    } else {
        Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 2, false, null));
    }
    // Container logs should be uploaded
    logFileStatusInLastCycle = verifyContainerLogs(logAggregationService, application, new ContainerId[] { container }, logFiles2, 3, true);
    for (String logFile : logFiles2) {
        Assert.assertTrue(logFileStatusInLastCycle.getLogFileTypesInLastCycle().contains(logFile));
    }
    // Make sure the std_final is not uploaded.
    Assert.assertFalse(logFileStatusInLastCycle.getLogFileTypesInLastCycle().contains("std_final"));
    Thread.sleep(2000);
    // create another logs
    String[] logFiles3 = new String[] { "stdout_2", "stderr_2", "syslog_2" };
    writeContainerLogs(appLogDir, container, logFiles3);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container, 0));
    dispatcher.await();
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application));
    if (retentionSizeLimitation) {
        Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 1, true, logFileStatusInLastCycle.getLogFilePathInLastCycle()));
    } else {
        Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 3, false, null));
    }
    // the app is finished. The log "std_final" should be aggregated this time.
    String[] logFiles3WithFinalLog = new String[] { "stdout_2", "stderr_2", "syslog_2", "std_final" };
    verifyContainerLogs(logAggregationService, application, new ContainerId[] { container }, logFiles3WithFinalLog, 4, true);
    logAggregationService.stop();
    assertEquals(0, logAggregationService.getNumAggregators());
}