Examples with ContainerManagementProtocol org.apache.hadoop.yarn.api.ContainerManagementProtocol used on opensource projects

Example 1 with ContainerManagementProtocol

use of org.apache.hadoop.yarn.api.ContainerManagementProtocol in project hadoop by apache.

the class TestContainerManagerSecurity method getContainerStatus.

private void getContainerStatus(YarnRPC rpc, org.apache.hadoop.yarn.api.records.Token nmToken, ContainerId containerId, ApplicationAttemptId appAttemptId, NodeId nodeId, boolean isExceptionExpected) throws Exception {
    List<ContainerId> containerIds = new ArrayList<ContainerId>();
    containerIds.add(containerId);
    GetContainerStatusesRequest request = GetContainerStatusesRequest.newInstance(containerIds);
    ContainerManagementProtocol proxy = null;
    try {
        proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, appAttemptId.toString());
        GetContainerStatusesResponse statuses = proxy.getContainerStatuses(request);
        if (statuses.getFailedRequests() != null && statuses.getFailedRequests().containsKey(containerId)) {
            parseAndThrowException(statuses.getFailedRequests().get(containerId).deSerialize());
        }
    } finally {
        if (proxy != null) {
            rpc.stopProxy(proxy, conf);
        }
    }
}

Example 2 with ContainerManagementProtocol

use of org.apache.hadoop.yarn.api.ContainerManagementProtocol in project hadoop by apache.

the class TestContainerManagerSecurity method testContainerTokenWithEpoch.

/**
   * This tests whether a containerId is serialized/deserialized with epoch.
   *
   * @throws IOException
   * @throws InterruptedException
   * @throws YarnException
   */
private void testContainerTokenWithEpoch(Configuration conf) throws IOException, InterruptedException, YarnException {
    LOG.info("Running test for serializing/deserializing containerIds");
    NMTokenSecretManagerInRM nmTokenSecretManagerInRM = yarnCluster.getResourceManager().getRMContext().getNMTokenSecretManager();
    ApplicationId appId = ApplicationId.newInstance(1, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 0);
    ContainerId cId = ContainerId.newContainerId(appAttemptId, (5L << 40) | 3L);
    NodeManager nm = yarnCluster.getNodeManager(0);
    NMTokenSecretManagerInNM nmTokenSecretManagerInNM = nm.getNMContext().getNMTokenSecretManager();
    String user = "test";
    waitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm);
    NodeId nodeId = nm.getNMContext().getNodeId();
    // Both id should be equal.
    Assert.assertEquals(nmTokenSecretManagerInNM.getCurrentKey().getKeyId(), nmTokenSecretManagerInRM.getCurrentKey().getKeyId());
    // Creating a normal Container Token
    RMContainerTokenSecretManager containerTokenSecretManager = yarnCluster.getResourceManager().getRMContext().getContainerTokenSecretManager();
    Resource r = Resource.newInstance(1230, 2);
    Token containerToken = containerTokenSecretManager.createContainerToken(cId, 0, nodeId, user, r, Priority.newInstance(0), 0);
    ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier();
    byte[] tokenIdentifierContent = containerToken.getIdentifier().array();
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(tokenIdentifierContent, tokenIdentifierContent.length);
    containerTokenIdentifier.readFields(dib);
    Assert.assertEquals(cId, containerTokenIdentifier.getContainerID());
    Assert.assertEquals(cId.toString(), containerTokenIdentifier.getContainerID().toString());
    Token nmToken = nmTokenSecretManagerInRM.createNMToken(appAttemptId, nodeId, user);
    YarnRPC rpc = YarnRPC.create(conf);
    testStartContainer(rpc, appAttemptId, nodeId, containerToken, nmToken, false);
    List<ContainerId> containerIds = new LinkedList<ContainerId>();
    containerIds.add(cId);
    ContainerManagementProtocol proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    GetContainerStatusesResponse res = proxy.getContainerStatuses(GetContainerStatusesRequest.newInstance(containerIds));
    Assert.assertNotNull(res.getContainerStatuses().get(0));
    Assert.assertEquals(cId, res.getContainerStatuses().get(0).getContainerId());
    Assert.assertEquals(cId.toString(), res.getContainerStatuses().get(0).getContainerId().toString());
}

Example 3 with ContainerManagementProtocol

use of org.apache.hadoop.yarn.api.ContainerManagementProtocol in project hadoop by apache.

the class TestContainerManagerSecurity method startContainer.

private void startContainer(final YarnRPC rpc, org.apache.hadoop.yarn.api.records.Token nmToken, org.apache.hadoop.yarn.api.records.Token containerToken, NodeId nodeId, String user) throws Exception {
    ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class);
    StartContainerRequest scRequest = StartContainerRequest.newInstance(context, containerToken);
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    ContainerManagementProtocol proxy = null;
    try {
        proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
        StartContainersResponse response = proxy.startContainers(allRequests);
        for (SerializedException ex : response.getFailedRequests().values()) {
            parseAndThrowException(ex.deSerialize());
        }
    } finally {
        if (proxy != null) {
            rpc.stopProxy(proxy, conf);
        }
    }
}

Example 4 with ContainerManagementProtocol

use of org.apache.hadoop.yarn.api.ContainerManagementProtocol in project hadoop by apache.

the class TestClientToAMTokens method testClientToAMTokens.

@Test
public void testClientToAMTokens() throws Exception {
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    // Set RPC engine to protobuf RPC engine
    RPC.setProtocolEngine(conf, CustomProtocol.class, ProtobufRpcEngine.class);
    UserGroupInformation.setConfiguration(conf);
    ContainerManagementProtocol containerManager = mock(ContainerManagementProtocol.class);
    StartContainersResponse mockResponse = mock(StartContainersResponse.class);
    when(containerManager.startContainers((StartContainersRequest) any())).thenReturn(mockResponse);
    final DrainDispatcher dispatcher = new DrainDispatcher();
    MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) {

        protected ClientRMService createClientRMService() {
            return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, getRMContext().getRMDelegationTokenSecretManager());
        }

        ;

        @Override
        protected Dispatcher createDispatcher() {
            return dispatcher;
        }

        @Override
        protected void doSecureLogin() throws IOException {
        }
    };
    rm.start();
    // Submit an app
    RMApp app = rm.submitApp(1024);
    // Set up a node.
    MockNM nm1 = rm.registerNode("localhost:1234", 3072);
    nm1.nodeHeartbeat(true);
    dispatcher.await();
    nm1.nodeHeartbeat(true);
    dispatcher.await();
    ApplicationAttemptId appAttempt = app.getCurrentAppAttempt().getAppAttemptId();
    final MockAM mockAM = new MockAM(rm.getRMContext(), rm.getApplicationMasterService(), app.getCurrentAppAttempt().getAppAttemptId());
    UserGroupInformation appUgi = UserGroupInformation.createRemoteUser(appAttempt.toString());
    RegisterApplicationMasterResponse response = appUgi.doAs(new PrivilegedAction<RegisterApplicationMasterResponse>() {

        @Override
        public RegisterApplicationMasterResponse run() {
            RegisterApplicationMasterResponse response = null;
            try {
                response = mockAM.registerAppAttempt();
            } catch (Exception e) {
                Assert.fail("Exception was not expected");
            }
            return response;
        }
    });
    // Get the app-report.
    GetApplicationReportRequest request = Records.newRecord(GetApplicationReportRequest.class);
    request.setApplicationId(app.getApplicationId());
    GetApplicationReportResponse reportResponse = rm.getClientRMService().getApplicationReport(request);
    ApplicationReport appReport = reportResponse.getApplicationReport();
    org.apache.hadoop.yarn.api.records.Token originalClientToAMToken = appReport.getClientToAMToken();
    // ClientToAMToken master key should have been received on register
    // application master response.
    Assert.assertNotNull(response.getClientToAMTokenMasterKey());
    Assert.assertTrue(response.getClientToAMTokenMasterKey().array().length > 0);
    // Start the AM with the correct shared-secret.
    ApplicationAttemptId appAttemptId = app.getAppAttempts().keySet().iterator().next();
    Assert.assertNotNull(appAttemptId);
    final CustomAM am = new CustomAM(appAttemptId, response.getClientToAMTokenMasterKey().array());
    am.init(conf);
    am.start();
    // Now the real test!
    // Set up clients to be able to pick up correct tokens.
    SecurityUtil.setSecurityInfoProviders(new CustomSecurityInfo());
    // Verify denial for unauthenticated user
    try {
        CustomProtocol client = RPC.getProxy(CustomProtocol.class, 1L, am.address, conf);
        client.ping(null, TestRpcBase.newEmptyRequest());
        fail("Access by unauthenticated user should fail!!");
    } catch (Exception e) {
        Assert.assertFalse(am.pinged);
    }
    Token<ClientToAMTokenIdentifier> token = ConverterUtils.convertFromYarn(originalClientToAMToken, am.address);
    // Verify denial for a malicious user with tampered ID
    verifyTokenWithTamperedID(conf, am, token);
    // Verify denial for a malicious user with tampered user-name
    verifyTokenWithTamperedUserName(conf, am, token);
    // Now for an authenticated user
    verifyValidToken(conf, am, token);
    // Verify for a new version token
    verifyNewVersionToken(conf, am, token, rm);
    am.stop();
    rm.stop();
}

Example 5 with ContainerManagementProtocol

use of org.apache.hadoop.yarn.api.ContainerManagementProtocol in project hadoop by apache.

the class TestNMProxy method testNMProxyRetry.

@Test(timeout = 20000)
public void testNMProxyRetry() throws Exception {
    conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_MAX_WAIT_MS, 10000);
    conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_RETRY_INTERVAL_MS, 100);
    StartContainersRequest allRequests = Records.newRecord(StartContainersRequest.class);
    ContainerManagementProtocol proxy = getNMProxy(conf);
    retryCount = 0;
    shouldThrowNMNotYetReadyException = false;
    proxy.startContainers(allRequests);
    Assert.assertEquals(5, retryCount);
    retryCount = 0;
    shouldThrowNMNotYetReadyException = false;
    proxy.stopContainers(Records.newRecord(StopContainersRequest.class));
    Assert.assertEquals(5, retryCount);
    retryCount = 0;
    shouldThrowNMNotYetReadyException = false;
    proxy.getContainerStatuses(Records.newRecord(GetContainerStatusesRequest.class));
    Assert.assertEquals(5, retryCount);
    retryCount = 0;
    shouldThrowNMNotYetReadyException = true;
    proxy.startContainers(allRequests);
    Assert.assertEquals(5, retryCount);
}