Search in sources :

Example 1 with ClientRMService

use of org.apache.hadoop.yarn.server.resourcemanager.ClientRMService in project hadoop by apache.

the class TestClientToAMTokens method testClientToAMTokens.

@Test
public void testClientToAMTokens() throws Exception {
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    // Set RPC engine to protobuf RPC engine
    RPC.setProtocolEngine(conf, CustomProtocol.class, ProtobufRpcEngine.class);
    UserGroupInformation.setConfiguration(conf);
    ContainerManagementProtocol containerManager = mock(ContainerManagementProtocol.class);
    StartContainersResponse mockResponse = mock(StartContainersResponse.class);
    when(containerManager.startContainers((StartContainersRequest) any())).thenReturn(mockResponse);
    final DrainDispatcher dispatcher = new DrainDispatcher();
    MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) {

        protected ClientRMService createClientRMService() {
            return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, getRMContext().getRMDelegationTokenSecretManager());
        }

        ;

        @Override
        protected Dispatcher createDispatcher() {
            return dispatcher;
        }

        @Override
        protected void doSecureLogin() throws IOException {
        }
    };
    rm.start();
    // Submit an app
    RMApp app = rm.submitApp(1024);
    // Set up a node.
    MockNM nm1 = rm.registerNode("localhost:1234", 3072);
    nm1.nodeHeartbeat(true);
    dispatcher.await();
    nm1.nodeHeartbeat(true);
    dispatcher.await();
    ApplicationAttemptId appAttempt = app.getCurrentAppAttempt().getAppAttemptId();
    final MockAM mockAM = new MockAM(rm.getRMContext(), rm.getApplicationMasterService(), app.getCurrentAppAttempt().getAppAttemptId());
    UserGroupInformation appUgi = UserGroupInformation.createRemoteUser(appAttempt.toString());
    RegisterApplicationMasterResponse response = appUgi.doAs(new PrivilegedAction<RegisterApplicationMasterResponse>() {

        @Override
        public RegisterApplicationMasterResponse run() {
            RegisterApplicationMasterResponse response = null;
            try {
                response = mockAM.registerAppAttempt();
            } catch (Exception e) {
                Assert.fail("Exception was not expected");
            }
            return response;
        }
    });
    // Get the app-report.
    GetApplicationReportRequest request = Records.newRecord(GetApplicationReportRequest.class);
    request.setApplicationId(app.getApplicationId());
    GetApplicationReportResponse reportResponse = rm.getClientRMService().getApplicationReport(request);
    ApplicationReport appReport = reportResponse.getApplicationReport();
    org.apache.hadoop.yarn.api.records.Token originalClientToAMToken = appReport.getClientToAMToken();
    // ClientToAMToken master key should have been received on register
    // application master response.
    Assert.assertNotNull(response.getClientToAMTokenMasterKey());
    Assert.assertTrue(response.getClientToAMTokenMasterKey().array().length > 0);
    // Start the AM with the correct shared-secret.
    ApplicationAttemptId appAttemptId = app.getAppAttempts().keySet().iterator().next();
    Assert.assertNotNull(appAttemptId);
    final CustomAM am = new CustomAM(appAttemptId, response.getClientToAMTokenMasterKey().array());
    am.init(conf);
    am.start();
    // Now the real test!
    // Set up clients to be able to pick up correct tokens.
    SecurityUtil.setSecurityInfoProviders(new CustomSecurityInfo());
    // Verify denial for unauthenticated user
    try {
        CustomProtocol client = RPC.getProxy(CustomProtocol.class, 1L, am.address, conf);
        client.ping(null, TestRpcBase.newEmptyRequest());
        fail("Access by unauthenticated user should fail!!");
    } catch (Exception e) {
        Assert.assertFalse(am.pinged);
    }
    Token<ClientToAMTokenIdentifier> token = ConverterUtils.convertFromYarn(originalClientToAMToken, am.address);
    // Verify denial for a malicious user with tampered ID
    verifyTokenWithTamperedID(conf, am, token);
    // Verify denial for a malicious user with tampered user-name
    verifyTokenWithTamperedUserName(conf, am, token);
    // Now for an authenticated user
    verifyValidToken(conf, am, token);
    // Verify for a new version token
    verifyNewVersionToken(conf, am, token, rm);
    am.stop();
    rm.stop();
}
Also used : DrainDispatcher(org.apache.hadoop.yarn.event.DrainDispatcher) RMApp(org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp) MockRMWithCustomAMLauncher(org.apache.hadoop.yarn.server.resourcemanager.MockRMWithCustomAMLauncher) MockNM(org.apache.hadoop.yarn.server.resourcemanager.MockNM) MockRM(org.apache.hadoop.yarn.server.resourcemanager.MockRM) UserGroupInformation(org.apache.hadoop.security.UserGroupInformation) GetApplicationReportRequest(org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest) StartContainersResponse(org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse) ApplicationAttemptId(org.apache.hadoop.yarn.api.records.ApplicationAttemptId) ServiceException(com.google.protobuf.ServiceException) SaslException(javax.security.sasl.SaslException) IOException(java.io.IOException) RemoteException(org.apache.hadoop.ipc.RemoteException) YarnRuntimeException(org.apache.hadoop.yarn.exceptions.YarnRuntimeException) ClientRMService(org.apache.hadoop.yarn.server.resourcemanager.ClientRMService) ApplicationReport(org.apache.hadoop.yarn.api.records.ApplicationReport) ClientToAMTokenIdentifier(org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier) ContainerManagementProtocol(org.apache.hadoop.yarn.api.ContainerManagementProtocol) RegisterApplicationMasterResponse(org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse) MockAM(org.apache.hadoop.yarn.server.resourcemanager.MockAM) GetApplicationReportResponse(org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse) Test(org.junit.Test)

Example 2 with ClientRMService

use of org.apache.hadoop.yarn.server.resourcemanager.ClientRMService in project hadoop by apache.

the class TestRMWebAppFairScheduler method mockRm.

private static ResourceManager mockRm(RMContext rmContext) throws IOException {
    ResourceManager rm = mock(ResourceManager.class);
    ResourceScheduler rs = mockFairScheduler();
    ClientRMService clientRMService = mockClientRMService(rmContext);
    when(rm.getResourceScheduler()).thenReturn(rs);
    when(rm.getRMContext()).thenReturn(rmContext);
    when(rm.getClientRMService()).thenReturn(clientRMService);
    return rm;
}
Also used : ResourceScheduler(org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler) ResourceManager(org.apache.hadoop.yarn.server.resourcemanager.ResourceManager) ClientRMService(org.apache.hadoop.yarn.server.resourcemanager.ClientRMService)

Example 3 with ClientRMService

use of org.apache.hadoop.yarn.server.resourcemanager.ClientRMService in project hadoop by apache.

the class TestRMWebAppFairScheduler method mockRmWithApps.

private static ResourceManager mockRmWithApps(RMContext rmContext) throws IOException {
    ResourceManager rm = mock(ResourceManager.class);
    ResourceScheduler rs = mockFairSchedulerWithoutApps(rmContext);
    ClientRMService clientRMService = mockClientRMService(rmContext);
    when(rm.getResourceScheduler()).thenReturn(rs);
    when(rm.getRMContext()).thenReturn(rmContext);
    when(rm.getClientRMService()).thenReturn(clientRMService);
    return rm;
}
Also used : ResourceScheduler(org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler) ResourceManager(org.apache.hadoop.yarn.server.resourcemanager.ResourceManager) ClientRMService(org.apache.hadoop.yarn.server.resourcemanager.ClientRMService)

Example 4 with ClientRMService

use of org.apache.hadoop.yarn.server.resourcemanager.ClientRMService in project hadoop by apache.

the class TestDelegationTokenRenewer method setUp.

@Before
public void setUp() throws Exception {
    counter = new AtomicInteger(0);
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    UserGroupInformation.setConfiguration(conf);
    eventQueue = new LinkedBlockingQueue<Event>();
    dispatcher = new AsyncDispatcher(eventQueue);
    Renewer.reset();
    delegationTokenRenewer = createNewDelegationTokenRenewer(conf, counter);
    RMContext mockContext = mock(RMContext.class);
    ClientRMService mockClientRMService = mock(ClientRMService.class);
    when(mockContext.getSystemCredentialsForApps()).thenReturn(new ConcurrentHashMap<ApplicationId, ByteBuffer>());
    when(mockContext.getDelegationTokenRenewer()).thenReturn(delegationTokenRenewer);
    when(mockContext.getDispatcher()).thenReturn(dispatcher);
    when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
    InetSocketAddress sockAddr = InetSocketAddress.createUnresolved("localhost", 1234);
    when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
    delegationTokenRenewer.setRMContext(mockContext);
    delegationTokenRenewer.init(conf);
    delegationTokenRenewer.start();
}
Also used : RMContext(org.apache.hadoop.yarn.server.resourcemanager.RMContext) AsyncDispatcher(org.apache.hadoop.yarn.event.AsyncDispatcher) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) InetSocketAddress(java.net.InetSocketAddress) Event(org.apache.hadoop.yarn.event.Event) RMAppEvent(org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) ByteBuffer(java.nio.ByteBuffer) DataInputByteBuffer(org.apache.hadoop.io.DataInputByteBuffer) ClientRMService(org.apache.hadoop.yarn.server.resourcemanager.ClientRMService) Before(org.junit.Before)

Example 5 with ClientRMService

use of org.apache.hadoop.yarn.server.resourcemanager.ClientRMService in project hadoop by apache.

the class TestRMWebServices method testAppsRace.

// Test the scenario where the RM removes an app just as we try to
// look at it in the apps list
@Test
public void testAppsRace() throws Exception {
    // mock up an RM that returns app reports for apps that don't exist
    // in the RMApps list
    ApplicationId appId = ApplicationId.newInstance(1, 1);
    ApplicationReport mockReport = mock(ApplicationReport.class);
    when(mockReport.getApplicationId()).thenReturn(appId);
    GetApplicationsResponse mockAppsResponse = mock(GetApplicationsResponse.class);
    when(mockAppsResponse.getApplicationList()).thenReturn(Arrays.asList(new ApplicationReport[] { mockReport }));
    ClientRMService mockClientSvc = mock(ClientRMService.class);
    when(mockClientSvc.getApplications(isA(GetApplicationsRequest.class), anyBoolean())).thenReturn(mockAppsResponse);
    ResourceManager mockRM = mock(ResourceManager.class);
    RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, null, null, null, null);
    when(mockRM.getRMContext()).thenReturn(rmContext);
    when(mockRM.getClientRMService()).thenReturn(mockClientSvc);
    rmContext.setNodeLabelManager(mock(RMNodeLabelsManager.class));
    RMWebServices webSvc = new RMWebServices(mockRM, new Configuration(), mock(HttpServletResponse.class));
    final Set<String> emptySet = Collections.unmodifiableSet(Collections.<String>emptySet());
    // verify we don't get any apps when querying
    HttpServletRequest mockHsr = mock(HttpServletRequest.class);
    AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null, null, null, null, null, null, null, null, emptySet, emptySet);
    assertTrue(appsInfo.getApps().isEmpty());
    // verify we don't get an NPE when specifying a final status query
    appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED", null, null, null, null, null, null, null, emptySet, emptySet);
    assertTrue(appsInfo.getApps().isEmpty());
}
Also used : Configuration(org.apache.hadoop.conf.Configuration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) HttpServletResponse(javax.servlet.http.HttpServletResponse) ResourceManager(org.apache.hadoop.yarn.server.resourcemanager.ResourceManager) GetApplicationsRequest(org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest) ClientRMService(org.apache.hadoop.yarn.server.resourcemanager.ClientRMService) ApplicationReport(org.apache.hadoop.yarn.api.records.ApplicationReport) HttpServletRequest(javax.servlet.http.HttpServletRequest) GetApplicationsResponse(org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) AppsInfo(org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo) RMContextImpl(org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl) RMNodeLabelsManager(org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager) Test(org.junit.Test)

Aggregations

ClientRMService (org.apache.hadoop.yarn.server.resourcemanager.ClientRMService)15 Test (org.junit.Test)9 ApplicationId (org.apache.hadoop.yarn.api.records.ApplicationId)7 ByteBuffer (java.nio.ByteBuffer)6 Configuration (org.apache.hadoop.conf.Configuration)6 RMContext (org.apache.hadoop.yarn.server.resourcemanager.RMContext)6 InetSocketAddress (java.net.InetSocketAddress)5 DataInputByteBuffer (org.apache.hadoop.io.DataInputByteBuffer)5 YarnConfiguration (org.apache.hadoop.yarn.conf.YarnConfiguration)5 Text (org.apache.hadoop.io.Text)4 Credentials (org.apache.hadoop.security.Credentials)4 ApplicationReport (org.apache.hadoop.yarn.api.records.ApplicationReport)4 IOException (java.io.IOException)3 GetApplicationReportRequest (org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest)3 GetApplicationReportResponse (org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse)3 ResourceManager (org.apache.hadoop.yarn.server.resourcemanager.ResourceManager)3 ServiceException (com.google.protobuf.ServiceException)2 SaslException (javax.security.sasl.SaslException)2 DelegationTokenIdentifier (org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier)2 RemoteException (org.apache.hadoop.ipc.RemoteException)2