Examples with OpportunisticContainerAllocator org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerAllocator used on opensource projects

Example 1 with OpportunisticContainerAllocator

use of org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerAllocator in project hadoop by apache.

the class NodeManager method serviceInit.

@Override
protected void serviceInit(Configuration conf) throws Exception {
    conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
    rmWorkPreservingRestartEnabled = conf.getBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, YarnConfiguration.DEFAULT_RM_WORK_PRESERVING_RECOVERY_ENABLED);
    try {
        initAndStartRecoveryStore(conf);
    } catch (IOException e) {
        String recoveryDirName = conf.get(YarnConfiguration.NM_RECOVERY_DIR);
        throw new YarnRuntimeException("Unable to initialize recovery directory at " + recoveryDirName, e);
    }
    NMContainerTokenSecretManager containerTokenSecretManager = new NMContainerTokenSecretManager(conf, nmStore);
    NMTokenSecretManagerInNM nmTokenSecretManager = new NMTokenSecretManagerInNM(nmStore);
    recoverTokens(nmTokenSecretManager, containerTokenSecretManager);
    this.aclsManager = new ApplicationACLsManager(conf);
    ContainerExecutor exec = ReflectionUtils.newInstance(conf.getClass(YarnConfiguration.NM_CONTAINER_EXECUTOR, DefaultContainerExecutor.class, ContainerExecutor.class), conf);
    try {
        exec.init();
    } catch (IOException e) {
        throw new YarnRuntimeException("Failed to initialize container executor", e);
    }
    DeletionService del = createDeletionService(exec);
    addService(del);
    // NodeManager level dispatcher
    this.dispatcher = new AsyncDispatcher("NM Event dispatcher");
    dirsHandler = new LocalDirsHandlerService(metrics);
    nodeHealthChecker = new NodeHealthCheckerService(getNodeHealthScriptRunner(conf), dirsHandler);
    addService(nodeHealthChecker);
    boolean isDistSchedulingEnabled = conf.getBoolean(YarnConfiguration.DIST_SCHEDULING_ENABLED, YarnConfiguration.DEFAULT_DIST_SCHEDULING_ENABLED);
    this.context = createNMContext(containerTokenSecretManager, nmTokenSecretManager, nmStore, isDistSchedulingEnabled, conf);
    ((NMContext) context).setContainerExecutor(exec);
    nodeLabelsProvider = createNodeLabelsProvider(conf);
    if (null == nodeLabelsProvider) {
        nodeStatusUpdater = createNodeStatusUpdater(context, dispatcher, nodeHealthChecker);
    } else {
        addIfService(nodeLabelsProvider);
        nodeStatusUpdater = createNodeStatusUpdater(context, dispatcher, nodeHealthChecker, nodeLabelsProvider);
    }
    nodeResourceMonitor = createNodeResourceMonitor();
    addService(nodeResourceMonitor);
    ((NMContext) context).setNodeResourceMonitor(nodeResourceMonitor);
    containerManager = createContainerManager(context, exec, del, nodeStatusUpdater, this.aclsManager, dirsHandler);
    addService(containerManager);
    ((NMContext) context).setContainerManager(containerManager);
    WebServer webServer = createWebServer(context, containerManager.getContainersMonitor(), this.aclsManager, dirsHandler);
    addService(webServer);
    ((NMContext) context).setWebServer(webServer);
    ((NMContext) context).setQueueableContainerAllocator(new OpportunisticContainerAllocator(context.getContainerTokenSecretManager()));
    dispatcher.register(ContainerManagerEventType.class, containerManager);
    dispatcher.register(NodeManagerEventType.class, this);
    addService(dispatcher);
    pauseMonitor = new JvmPauseMonitor();
    addService(pauseMonitor);
    metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
    DefaultMetricsSystem.initialize("NodeManager");
    if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
        this.nmCollectorService = createNMCollectorService(context);
        addService(nmCollectorService);
    }
    // StatusUpdater should be added last so that it get started last 
    // so that we make sure everything is up before registering with RM. 
    addService(nodeStatusUpdater);
    ((NMContext) context).setNodeStatusUpdater(nodeStatusUpdater);
    super.serviceInit(conf);
// TODO add local dirs to del
}

Example 2 with OpportunisticContainerAllocator

use of org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerAllocator in project hadoop by apache.

the class TestDistributedScheduler method setup.

private RequestInterceptor setup(Configuration conf, DistributedScheduler distributedScheduler) {
    NodeStatusUpdater nodeStatusUpdater = Mockito.mock(NodeStatusUpdater.class);
    Mockito.when(nodeStatusUpdater.getRMIdentifier()).thenReturn(12345l);
    NMContainerTokenSecretManager nmContainerTokenSecretManager = new NMContainerTokenSecretManager(conf);
    MasterKey mKey = new MasterKey() {

        @Override
        public int getKeyId() {
            return 1;
        }

        @Override
        public void setKeyId(int keyId) {
        }

        @Override
        public ByteBuffer getBytes() {
            return ByteBuffer.allocate(8);
        }

        @Override
        public void setBytes(ByteBuffer bytes) {
        }
    };
    nmContainerTokenSecretManager.setMasterKey(mKey);
    OpportunisticContainerAllocator containerAllocator = new OpportunisticContainerAllocator(nmContainerTokenSecretManager);
    NMTokenSecretManagerInNM nmTokenSecretManagerInNM = new NMTokenSecretManagerInNM();
    nmTokenSecretManagerInNM.setMasterKey(mKey);
    distributedScheduler.initLocal(1234, ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1), containerAllocator, nmTokenSecretManagerInNM, "test");
    RequestInterceptor finalReqIntcptr = Mockito.mock(RequestInterceptor.class);
    distributedScheduler.setNextInterceptor(finalReqIntcptr);
    return finalReqIntcptr;
}