Examples with NMContainerTokenSecretManager org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager used on opensource projects

Example 1 with NMContainerTokenSecretManager

use of org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager in project hadoop by apache.

the class NodeManager method serviceInit.

@Override
protected void serviceInit(Configuration conf) throws Exception {
    conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
    rmWorkPreservingRestartEnabled = conf.getBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, YarnConfiguration.DEFAULT_RM_WORK_PRESERVING_RECOVERY_ENABLED);
    try {
        initAndStartRecoveryStore(conf);
    } catch (IOException e) {
        String recoveryDirName = conf.get(YarnConfiguration.NM_RECOVERY_DIR);
        throw new YarnRuntimeException("Unable to initialize recovery directory at " + recoveryDirName, e);
    }
    NMContainerTokenSecretManager containerTokenSecretManager = new NMContainerTokenSecretManager(conf, nmStore);
    NMTokenSecretManagerInNM nmTokenSecretManager = new NMTokenSecretManagerInNM(nmStore);
    recoverTokens(nmTokenSecretManager, containerTokenSecretManager);
    this.aclsManager = new ApplicationACLsManager(conf);
    ContainerExecutor exec = ReflectionUtils.newInstance(conf.getClass(YarnConfiguration.NM_CONTAINER_EXECUTOR, DefaultContainerExecutor.class, ContainerExecutor.class), conf);
    try {
        exec.init();
    } catch (IOException e) {
        throw new YarnRuntimeException("Failed to initialize container executor", e);
    }
    DeletionService del = createDeletionService(exec);
    addService(del);
    // NodeManager level dispatcher
    this.dispatcher = new AsyncDispatcher("NM Event dispatcher");
    dirsHandler = new LocalDirsHandlerService(metrics);
    nodeHealthChecker = new NodeHealthCheckerService(getNodeHealthScriptRunner(conf), dirsHandler);
    addService(nodeHealthChecker);
    boolean isDistSchedulingEnabled = conf.getBoolean(YarnConfiguration.DIST_SCHEDULING_ENABLED, YarnConfiguration.DEFAULT_DIST_SCHEDULING_ENABLED);
    this.context = createNMContext(containerTokenSecretManager, nmTokenSecretManager, nmStore, isDistSchedulingEnabled, conf);
    ((NMContext) context).setContainerExecutor(exec);
    nodeLabelsProvider = createNodeLabelsProvider(conf);
    if (null == nodeLabelsProvider) {
        nodeStatusUpdater = createNodeStatusUpdater(context, dispatcher, nodeHealthChecker);
    } else {
        addIfService(nodeLabelsProvider);
        nodeStatusUpdater = createNodeStatusUpdater(context, dispatcher, nodeHealthChecker, nodeLabelsProvider);
    }
    nodeResourceMonitor = createNodeResourceMonitor();
    addService(nodeResourceMonitor);
    ((NMContext) context).setNodeResourceMonitor(nodeResourceMonitor);
    containerManager = createContainerManager(context, exec, del, nodeStatusUpdater, this.aclsManager, dirsHandler);
    addService(containerManager);
    ((NMContext) context).setContainerManager(containerManager);
    WebServer webServer = createWebServer(context, containerManager.getContainersMonitor(), this.aclsManager, dirsHandler);
    addService(webServer);
    ((NMContext) context).setWebServer(webServer);
    ((NMContext) context).setQueueableContainerAllocator(new OpportunisticContainerAllocator(context.getContainerTokenSecretManager()));
    dispatcher.register(ContainerManagerEventType.class, containerManager);
    dispatcher.register(NodeManagerEventType.class, this);
    addService(dispatcher);
    pauseMonitor = new JvmPauseMonitor();
    addService(pauseMonitor);
    metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
    DefaultMetricsSystem.initialize("NodeManager");
    if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
        this.nmCollectorService = createNMCollectorService(context);
        addService(nmCollectorService);
    }
    // StatusUpdater should be added last so that it get started last 
    // so that we make sure everything is up before registering with RM. 
    addService(nodeStatusUpdater);
    ((NMContext) context).setNodeStatusUpdater(nodeStatusUpdater);
    super.serviceInit(conf);
// TODO add local dirs to del
}

Example 2 with NMContainerTokenSecretManager

use of org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager in project hadoop by apache.

the class TestDistributedScheduler method setup.

private RequestInterceptor setup(Configuration conf, DistributedScheduler distributedScheduler) {
    NodeStatusUpdater nodeStatusUpdater = Mockito.mock(NodeStatusUpdater.class);
    Mockito.when(nodeStatusUpdater.getRMIdentifier()).thenReturn(12345l);
    NMContainerTokenSecretManager nmContainerTokenSecretManager = new NMContainerTokenSecretManager(conf);
    MasterKey mKey = new MasterKey() {

        @Override
        public int getKeyId() {
            return 1;
        }

        @Override
        public void setKeyId(int keyId) {
        }

        @Override
        public ByteBuffer getBytes() {
            return ByteBuffer.allocate(8);
        }

        @Override
        public void setBytes(ByteBuffer bytes) {
        }
    };
    nmContainerTokenSecretManager.setMasterKey(mKey);
    OpportunisticContainerAllocator containerAllocator = new OpportunisticContainerAllocator(nmContainerTokenSecretManager);
    NMTokenSecretManagerInNM nmTokenSecretManagerInNM = new NMTokenSecretManagerInNM();
    nmTokenSecretManagerInNM.setMasterKey(mKey);
    distributedScheduler.initLocal(1234, ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1), containerAllocator, nmTokenSecretManagerInNM, "test");
    RequestInterceptor finalReqIntcptr = Mockito.mock(RequestInterceptor.class);
    distributedScheduler.setNextInterceptor(finalReqIntcptr);
    return finalReqIntcptr;
}

Example 3 with NMContainerTokenSecretManager

use of org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager in project hadoop by apache.

the class TestEventFlow method testSuccessfulContainerLaunch.

@Test
public void testSuccessfulContainerLaunch() throws InterruptedException, IOException, YarnException {
    FileContext localFS = FileContext.getLocalFSFileContext();
    localFS.delete(new Path(localDir.getAbsolutePath()), true);
    localFS.delete(new Path(localLogDir.getAbsolutePath()), true);
    localFS.delete(new Path(remoteLogDir.getAbsolutePath()), true);
    localDir.mkdir();
    localLogDir.mkdir();
    remoteLogDir.mkdir();
    YarnConfiguration conf = new YarnConfiguration();
    Context context = new NMContext(new NMContainerTokenSecretManager(conf), new NMTokenSecretManagerInNM(), null, null, new NMNullStateStoreService(), false, conf) {

        @Override
        public int getHttpPort() {
            return 1234;
        }
    };
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "0.0.0.0:" + ServerSocketUtil.getPort(8040, 10));
    ContainerExecutor exec = new DefaultContainerExecutor();
    exec.setConf(conf);
    DeletionService del = new DeletionService(exec);
    Dispatcher dispatcher = new AsyncDispatcher();
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(NodeManager.getNodeHealthScriptRunner(conf), dirsHandler);
    healthChecker.init(conf);
    NodeManagerMetrics metrics = NodeManagerMetrics.create();
    NodeStatusUpdater nodeStatusUpdater = new NodeStatusUpdaterImpl(context, dispatcher, healthChecker, metrics) {

        @Override
        protected ResourceTracker getRMClient() {
            return new LocalRMInterface();
        }

        ;

        @Override
        protected void stopRMProxy() {
            return;
        }

        @Override
        protected void startStatusUpdater() {
            // Don't start any updating thread.
            return;
        }

        @Override
        public long getRMIdentifier() {
            return SIMULATED_RM_IDENTIFIER;
        }
    };
    DummyContainerManager containerManager = new DummyContainerManager(context, exec, del, nodeStatusUpdater, metrics, dirsHandler);
    nodeStatusUpdater.init(conf);
    ((NMContext) context).setContainerManager(containerManager);
    nodeStatusUpdater.start();
    ((NMContext) context).setNodeStatusUpdater(nodeStatusUpdater);
    containerManager.init(conf);
    containerManager.start();
    ContainerLaunchContext launchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    ApplicationId applicationId = ApplicationId.newInstance(0, 0);
    ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0);
    ContainerId cID = ContainerId.newContainerId(applicationAttemptId, 0);
    String user = "testing";
    StartContainerRequest scRequest = StartContainerRequest.newInstance(launchContext, TestContainerManager.createContainerToken(cID, SIMULATED_RM_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager()));
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    BaseContainerManagerTest.waitForContainerState(containerManager, cID, Arrays.asList(ContainerState.RUNNING, ContainerState.SCHEDULED), 20);
    List<ContainerId> containerIds = new ArrayList<ContainerId>();
    containerIds.add(cID);
    StopContainersRequest stopRequest = StopContainersRequest.newInstance(containerIds);
    containerManager.stopContainers(stopRequest);
    BaseContainerManagerTest.waitForContainerState(containerManager, cID, ContainerState.COMPLETE);
    containerManager.stop();
}

Example 4 with NMContainerTokenSecretManager

use of org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager in project hadoop by apache.

the class TestContainerManagerRecovery method createContext.

private NMContext createContext(Configuration conf, NMStateStoreService stateStore) {
    NMContext context = new NMContext(new NMContainerTokenSecretManager(conf), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore, false, conf) {

        public int getHttpPort() {
            return HTTP_PORT;
        }
    };
    // simulate registration with RM
    MasterKey masterKey = new MasterKeyPBImpl();
    masterKey.setKeyId(123);
    masterKey.setBytes(ByteBuffer.wrap(new byte[] { new Integer(123).byteValue() }));
    context.getContainerTokenSecretManager().setMasterKey(masterKey);
    context.getNMTokenSecretManager().setMasterKey(masterKey);
    return context;
}

Example 5 with NMContainerTokenSecretManager

use of org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager in project hadoop by apache.

the class TestNodeStatusUpdater method testCompletedContainerStatusBackup.

/**
   * Test completed containerStatus get back up when heart beat lost, and will
   * be sent via next heart beat.
   */
@Test(timeout = 200000)
public void testCompletedContainerStatusBackup() throws Exception {
    nm = new NodeManager() {

        @Override
        protected NodeStatusUpdater createNodeStatusUpdater(Context context, Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
            MyNodeStatusUpdater2 myNodeStatusUpdater = new MyNodeStatusUpdater2(context, dispatcher, healthChecker, metrics);
            return myNodeStatusUpdater;
        }

        @Override
        protected NMContext createNMContext(NMContainerTokenSecretManager containerTokenSecretManager, NMTokenSecretManagerInNM nmTokenSecretManager, NMStateStoreService store, boolean isDistributedSchedulingEnabled, Configuration config) {
            return new MyNMContext(containerTokenSecretManager, nmTokenSecretManager, config);
        }
    };
    YarnConfiguration conf = createNMConfig();
    nm.init(conf);
    nm.start();
    int waitCount = 0;
    while (heartBeatID <= 4 && waitCount++ != 20) {
        Thread.sleep(500);
    }
    if (heartBeatID <= 4) {
        Assert.fail("Failed to get all heartbeats in time, " + "heartbeatID:" + heartBeatID);
    }
    if (assertionFailedInThread.get()) {
        Assert.fail("ContainerStatus Backup failed");
    }
    Assert.assertNotNull(nm.getNMContext().getSystemCredentialsForApps().get(ApplicationId.newInstance(1234, 1)).getToken(new Text("token1")));
    nm.stop();
}