Examples with LocalDirsHandlerService org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService used on opensource projects

Example 6 with LocalDirsHandlerService

use of org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService in project hadoop by apache.

the class TestLogAggregationService method testInvalidThreadPoolSizeValue.

private void testInvalidThreadPoolSizeValue(final String threadPoolSize) throws IOException {
    Supplier<Boolean> isInputInvalid = new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            try {
                int value = Integer.parseInt(threadPoolSize);
                return value <= 0;
            } catch (NumberFormatException ex) {
                return true;
            }
        }
    };
    assertTrue("The thread pool size must be invalid to use with this " + "method", isInputInvalid.get());
    // store configured thread pool size temporarily for restoration
    int initThreadPoolSize = conf.getInt(YarnConfiguration.NM_LOG_AGGREGATION_THREAD_POOL_SIZE, YarnConfiguration.DEFAULT_NM_LOG_AGGREGATION_THREAD_POOL_SIZE);
    conf.set(YarnConfiguration.NM_LOG_AGGREGATION_THREAD_POOL_SIZE, threadPoolSize);
    DeletionService delSrvc = mock(DeletionService.class);
    LocalDirsHandlerService dirSvc = mock(LocalDirsHandlerService.class);
    when(dirSvc.getLogDirs()).thenThrow(new RuntimeException());
    LogAggregationService logAggregationService = new LogAggregationService(dispatcher, this.context, delSrvc, dirSvc);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    ThreadPoolExecutor executorService = (ThreadPoolExecutor) logAggregationService.threadPool;
    assertEquals("The thread pool size should be set to the value of YARN" + ".DEFAULT_NM_LOG_AGGREGATION_THREAD_POOL_SIZE because the configured " + " thread pool size is " + "invalid.", YarnConfiguration.DEFAULT_NM_LOG_AGGREGATION_THREAD_POOL_SIZE, executorService.getMaximumPoolSize());
    logAggregationService.stop();
    logAggregationService.close();
    // retore original configuration to aviod side effects
    conf.setInt(YarnConfiguration.NM_LOG_AGGREGATION_THREAD_POOL_SIZE, initThreadPoolSize);
}

Example 7 with LocalDirsHandlerService

use of org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService in project hadoop by apache.

the class TestLogAggregationService method testStopAfterError.

@Test(timeout = 20000)
public void testStopAfterError() throws Exception {
    DeletionService delSrvc = mock(DeletionService.class);
    // get the AppLogAggregationImpl thread to crash
    LocalDirsHandlerService mockedDirSvc = mock(LocalDirsHandlerService.class);
    when(mockedDirSvc.getLogDirs()).thenThrow(new RuntimeException());
    LogAggregationService logAggregationService = new LogAggregationService(dispatcher, this.context, delSrvc, mockedDirSvc);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1);
    LogAggregationContext contextWithAllContainers = Records.newRecord(LogAggregationContext.class);
    contextWithAllContainers.setLogAggregationPolicyClassName(AllContainerLogAggregationPolicy.class.getName());
    logAggregationService.handle(new LogHandlerAppStartedEvent(application1, this.user, null, this.acls, contextWithAllContainers));
    logAggregationService.stop();
    assertEquals(0, logAggregationService.getNumAggregators());
    logAggregationService.close();
}

Example 8 with LocalDirsHandlerService

use of org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService in project hadoop by apache.

the class TestLogAggregationService method testLocalFileDeletionOnDiskFull.

@Test
public void testLocalFileDeletionOnDiskFull() throws Exception {
    this.delSrvc = new DeletionService(createContainerExecutor());
    delSrvc = spy(delSrvc);
    this.delSrvc.init(conf);
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.getAbsolutePath());
    List<String> logDirs = super.dirsHandler.getLogDirs();
    LocalDirsHandlerService dirsHandler = spy(super.dirsHandler);
    // Simulate disk being full by returning no good log dirs but having a
    // directory in full log dirs.
    when(dirsHandler.getLogDirs()).thenReturn(new ArrayList<String>());
    when(dirsHandler.getLogDirsForRead()).thenReturn(logDirs);
    LogAggregationService logAggregationService = spy(new LogAggregationService(dispatcher, this.context, this.delSrvc, dirsHandler));
    verifyLocalFileDeletion(logAggregationService);
}

Example 9 with LocalDirsHandlerService

use of org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService in project hadoop by apache.

the class TestLogAggregationService method testFixedSizeThreadPool.

@Test(timeout = 30000)
public void testFixedSizeThreadPool() throws Exception {
    // store configured thread pool size temporarily for restoration
    int initThreadPoolSize = conf.getInt(YarnConfiguration.NM_LOG_AGGREGATION_THREAD_POOL_SIZE, YarnConfiguration.DEFAULT_NM_LOG_AGGREGATION_THREAD_POOL_SIZE);
    int threadPoolSize = 3;
    conf.setInt(YarnConfiguration.NM_LOG_AGGREGATION_THREAD_POOL_SIZE, threadPoolSize);
    DeletionService delSrvc = mock(DeletionService.class);
    LocalDirsHandlerService dirSvc = mock(LocalDirsHandlerService.class);
    when(dirSvc.getLogDirs()).thenThrow(new RuntimeException());
    LogAggregationService logAggregationService = new LogAggregationService(dispatcher, this.context, delSrvc, dirSvc);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    ExecutorService executorService = logAggregationService.threadPool;
    // used to block threads in the thread pool because main thread always
    // acquires the write lock first.
    final ReadWriteLock rwLock = new ReentrantReadWriteLock();
    final Lock rLock = rwLock.readLock();
    final Lock wLock = rwLock.writeLock();
    try {
        wLock.lock();
        Runnable runnable = new Runnable() {

            @Override
            public void run() {
                try {
                    // threads in the thread pool running this will be blocked
                    rLock.tryLock(35000, TimeUnit.MILLISECONDS);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                } finally {
                    rLock.unlock();
                }
            }
        };
        // created in the thread pool, each of which is blocked on the read lock.
        for (int i = 0; i < threadPoolSize + 1; i++) {
            executorService.submit(runnable);
        }
        // count the number of current running LogAggregationService threads
        int runningThread = ((ThreadPoolExecutor) executorService).getActiveCount();
        assertEquals(threadPoolSize, runningThread);
    } finally {
        wLock.unlock();
    }
    logAggregationService.stop();
    logAggregationService.close();
    // restore the original configurations to avoid side effects
    conf.setInt(YarnConfiguration.NM_LOG_AGGREGATION_THREAD_POOL_SIZE, initThreadPoolSize);
}

Example 10 with LocalDirsHandlerService

use of org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService in project hadoop by apache.

the class ContainerLogsUtils method getContainerLogFile.

/**
   * Finds the log file with the given filename for the given container.
   */
public static File getContainerLogFile(ContainerId containerId, String fileName, String remoteUser, Context context) throws YarnException {
    Container container = context.getContainers().get(containerId);
    Application application = getApplicationForContainer(containerId, context);
    checkAccess(remoteUser, application, context);
    if (container != null) {
        checkState(container.getContainerState());
    }
    try {
        LocalDirsHandlerService dirsHandler = context.getLocalDirsHandler();
        String relativeContainerLogDir = ContainerLaunch.getRelativeContainerLogDir(application.getAppId().toString(), containerId.toString());
        Path logPath = dirsHandler.getLogPathToRead(relativeContainerLogDir + Path.SEPARATOR + fileName);
        URI logPathURI = new File(logPath.toString()).toURI();
        File logFile = new File(logPathURI.getPath());
        return logFile;
    } catch (IOException e) {
        LOG.warn("Failed to find log file", e);
        throw new NotFoundException("Cannot find this log on the local disk.");
    }
}