Search in sources :

Example 6 with ApplicationMasterProtocol

use of org.apache.hadoop.yarn.api.ApplicationMasterProtocol in project hadoop by apache.

the class TestAMRMClient method testAMRMClientOnAMRMTokenRollOver.

@Test(timeout = 60000)
public void testAMRMClientOnAMRMTokenRollOver() throws YarnException, IOException {
    AMRMClient<ContainerRequest> amClient = null;
    try {
        AMRMTokenSecretManager amrmTokenSecretManager = yarnCluster.getResourceManager().getRMContext().getAMRMTokenSecretManager();
        // start am rm client
        amClient = AMRMClient.<ContainerRequest>createAMRMClient();
        amClient.init(conf);
        amClient.start();
        Long startTime = System.currentTimeMillis();
        amClient.registerApplicationMaster("Host", 10000, "");
        org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken_1 = getAMRMToken();
        Assert.assertNotNull(amrmToken_1);
        Assert.assertEquals(amrmToken_1.decodeIdentifier().getKeyId(), amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId());
        // At mean time, the old AMRMToken should continue to work
        while (System.currentTimeMillis() - startTime < rolling_interval_sec * 1000) {
            amClient.allocate(0.1f);
            sleep(1000);
        }
        amClient.allocate(0.1f);
        org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken_2 = getAMRMToken();
        Assert.assertNotNull(amrmToken_2);
        Assert.assertEquals(amrmToken_2.decodeIdentifier().getKeyId(), amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId());
        Assert.assertNotEquals(amrmToken_1, amrmToken_2);
        // can do the allocate call with latest AMRMToken
        AllocateResponse response = amClient.allocate(0.1f);
        // Verify latest AMRMToken can be used to send allocation request.
        UserGroupInformation testUser1 = UserGroupInformation.createRemoteUser("testUser1");
        AMRMTokenIdentifierForTest newVersionTokenIdentifier = new AMRMTokenIdentifierForTest(amrmToken_2.decodeIdentifier(), "message");
        Assert.assertEquals("Message is changed after set to newVersionTokenIdentifier", "message", newVersionTokenIdentifier.getMessage());
        org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> newVersionToken = new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(newVersionTokenIdentifier.getBytes(), amrmTokenSecretManager.retrievePassword(newVersionTokenIdentifier), newVersionTokenIdentifier.getKind(), new Text());
        SecurityUtil.setTokenService(newVersionToken, yarnCluster.getResourceManager().getApplicationMasterService().getBindAddress());
        testUser1.addToken(newVersionToken);
        AllocateRequest request = Records.newRecord(AllocateRequest.class);
        request.setResponseId(response.getResponseId());
        testUser1.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {

            @Override
            public ApplicationMasterProtocol run() {
                return (ApplicationMasterProtocol) YarnRPC.create(conf).getProxy(ApplicationMasterProtocol.class, yarnCluster.getResourceManager().getApplicationMasterService().getBindAddress(), conf);
            }
        }).allocate(request);
        // and can not use this rolled-over token to make a allocate all.
        while (true) {
            if (amrmToken_2.decodeIdentifier().getKeyId() != amrmTokenSecretManager.getCurrnetMasterKeyData().getMasterKey().getKeyId()) {
                if (amrmTokenSecretManager.getNextMasterKeyData() == null) {
                    break;
                } else if (amrmToken_2.decodeIdentifier().getKeyId() != amrmTokenSecretManager.getNextMasterKeyData().getMasterKey().getKeyId()) {
                    break;
                }
            }
            amClient.allocate(0.1f);
            sleep(1000);
        }
        try {
            UserGroupInformation testUser2 = UserGroupInformation.createRemoteUser("testUser2");
            SecurityUtil.setTokenService(amrmToken_2, yarnCluster.getResourceManager().getApplicationMasterService().getBindAddress());
            testUser2.addToken(amrmToken_2);
            testUser2.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {

                @Override
                public ApplicationMasterProtocol run() {
                    return (ApplicationMasterProtocol) YarnRPC.create(conf).getProxy(ApplicationMasterProtocol.class, yarnCluster.getResourceManager().getApplicationMasterService().getBindAddress(), conf);
                }
            }).allocate(Records.newRecord(AllocateRequest.class));
            Assert.fail("The old Token should not work");
        } catch (Exception ex) {
            Assert.assertTrue(ex instanceof InvalidToken);
            Assert.assertTrue(ex.getMessage().contains("Invalid AMRMToken from " + amrmToken_2.decodeIdentifier().getApplicationAttemptId()));
        }
        amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null);
    } finally {
        if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
            amClient.stop();
        }
    }
}
Also used : AllocateRequest(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest) InvalidToken(org.apache.hadoop.security.token.SecretManager.InvalidToken) Text(org.apache.hadoop.io.Text) ApplicationMasterProtocol(org.apache.hadoop.yarn.api.ApplicationMasterProtocol) AMRMTokenSecretManager(org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager) YarnException(org.apache.hadoop.yarn.exceptions.YarnException) IOException(java.io.IOException) InvalidContainerRequestException(org.apache.hadoop.yarn.client.api.InvalidContainerRequestException) AllocateResponse(org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse) AMRMTokenIdentifier(org.apache.hadoop.yarn.security.AMRMTokenIdentifier) PrivilegedAction(java.security.PrivilegedAction) InvalidToken(org.apache.hadoop.security.token.SecretManager.InvalidToken) ContainerRequest(org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest) UserGroupInformation(org.apache.hadoop.security.UserGroupInformation) Test(org.junit.Test)

Example 7 with ApplicationMasterProtocol

use of org.apache.hadoop.yarn.api.ApplicationMasterProtocol in project hadoop by apache.

the class TestAMRMClient method testAllocation.

private void testAllocation(final AMRMClientImpl<ContainerRequest> amClient) throws YarnException, IOException {
    // setup container request
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    assertNumContainers(amClient, 0, 2, 2, 2, 3, 0);
    int containersRequestedAny = 2;
    // RM should allocate container within 2 calls to allocate()
    int allocatedContainerCount = 0;
    int iterationsLeft = 3;
    Set<ContainerId> releases = new TreeSet<ContainerId>();
    amClient.getNMTokenCache().clearCache();
    Assert.assertEquals(0, amClient.getNMTokenCache().numberOfTokensInCache());
    HashMap<String, Token> receivedNMTokens = new HashMap<String, Token>();
    while (allocatedContainerCount < containersRequestedAny && iterationsLeft-- > 0) {
        AllocateResponse allocResponse = amClient.allocate(0.1f);
        assertEquals(0, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        assertEquals(nodeCount, amClient.getClusterNodeCount());
        allocatedContainerCount += allocResponse.getAllocatedContainers().size();
        for (Container container : allocResponse.getAllocatedContainers()) {
            ContainerId rejectContainerId = container.getId();
            releases.add(rejectContainerId);
            amClient.releaseAssignedContainer(rejectContainerId);
        }
        for (NMToken token : allocResponse.getNMTokens()) {
            String nodeID = token.getNodeId().toString();
            if (receivedNMTokens.containsKey(nodeID)) {
                Assert.fail("Received token again for : " + nodeID);
            }
            receivedNMTokens.put(nodeID, token.getToken());
        }
        if (allocatedContainerCount < containersRequestedAny) {
            // let NM heartbeat to RM and trigger allocations
            triggerSchedulingWithNMHeartBeat();
        }
    }
    // Should receive atleast 1 token
    Assert.assertTrue(receivedNMTokens.size() > 0 && receivedNMTokens.size() <= nodeCount);
    assertEquals(allocatedContainerCount, containersRequestedAny);
    assertEquals(2, amClient.release.size());
    assertEquals(0, amClient.ask.size());
    // need to tell the AMRMClient that we dont need these resources anymore
    amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    assertEquals(3, amClient.ask.size());
    // send 0 container count request for resources that are no longer needed
    ResourceRequest snoopRequest = amClient.ask.iterator().next();
    assertEquals(0, snoopRequest.getNumContainers());
    // test RPC exception handling
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    amClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
    snoopRequest = amClient.ask.iterator().next();
    assertEquals(2, snoopRequest.getNumContainers());
    ApplicationMasterProtocol realRM = amClient.rmClient;
    try {
        ApplicationMasterProtocol mockRM = mock(ApplicationMasterProtocol.class);
        when(mockRM.allocate(any(AllocateRequest.class))).thenAnswer(new Answer<AllocateResponse>() {

            public AllocateResponse answer(InvocationOnMock invocation) throws Exception {
                amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
                amClient.removeContainerRequest(new ContainerRequest(capability, nodes, racks, priority));
                throw new Exception();
            }
        });
        amClient.rmClient = mockRM;
        amClient.allocate(0.1f);
    } catch (Exception ioe) {
    } finally {
        amClient.rmClient = realRM;
    }
    assertEquals(2, amClient.release.size());
    assertEquals(3, amClient.ask.size());
    snoopRequest = amClient.ask.iterator().next();
    // verify that the remove request made in between makeRequest and allocate 
    // has not been lost
    assertEquals(0, snoopRequest.getNumContainers());
    waitForContainerCompletion(3, amClient, releases);
}
Also used : HashMap(java.util.HashMap) AllocateRequest(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest) InvalidToken(org.apache.hadoop.security.token.SecretManager.InvalidToken) ApplicationMasterProtocol(org.apache.hadoop.yarn.api.ApplicationMasterProtocol) YarnException(org.apache.hadoop.yarn.exceptions.YarnException) IOException(java.io.IOException) InvalidContainerRequestException(org.apache.hadoop.yarn.client.api.InvalidContainerRequestException) AllocateResponse(org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse) TreeSet(java.util.TreeSet) InvocationOnMock(org.mockito.invocation.InvocationOnMock) ContainerRequest(org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest)

Example 8 with ApplicationMasterProtocol

use of org.apache.hadoop.yarn.api.ApplicationMasterProtocol in project hadoop by apache.

the class TestAMRMClientOnRMRestart method testAMRMClientOnAMRMTokenRollOverOnRMRestart.

// Test verify for AM issued with rolled-over AMRMToken
// is still able to communicate with restarted RM.
@Test(timeout = 30000)
public void testAMRMClientOnAMRMTokenRollOverOnRMRestart() throws Exception {
    conf.setLong(YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, rolling_interval_sec);
    conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    // start first RM
    MyResourceManager2 rm1 = new MyResourceManager2(conf, memStore);
    rm1.start();
    DrainDispatcher dispatcher = (DrainDispatcher) rm1.getRMContext().getDispatcher();
    Long startTime = System.currentTimeMillis();
    // Submit the application
    RMApp app = rm1.submitApp(1024);
    dispatcher.await();
    MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();
    // Node heartbeat
    nm1.nodeHeartbeat(true);
    dispatcher.await();
    ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt().getAppAttemptId();
    rm1.sendAMLaunched(appAttemptId);
    dispatcher.await();
    AMRMTokenSecretManager amrmTokenSecretManagerForRM1 = rm1.getRMContext().getAMRMTokenSecretManager();
    org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> token = amrmTokenSecretManagerForRM1.createAndGetAMRMToken(appAttemptId);
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    ugi.addTokenIdentifier(token.decodeIdentifier());
    AMRMClient<ContainerRequest> amClient = new MyAMRMClientImpl(rm1);
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("h1", 10000, "");
    amClient.allocate(0.1f);
    // At mean time, the old AMRMToken should continue to work
    while (System.currentTimeMillis() - startTime < rolling_interval_sec * 1000) {
        amClient.allocate(0.1f);
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
        // DO NOTHING
        }
    }
    Assert.assertTrue(amrmTokenSecretManagerForRM1.getMasterKey().getMasterKey().getKeyId() != token.decodeIdentifier().getKeyId());
    amClient.allocate(0.1f);
    // active the nextMasterKey, and replace the currentMasterKey
    org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> newToken = amrmTokenSecretManagerForRM1.createAndGetAMRMToken(appAttemptId);
    int waitCount = 0;
    while (waitCount++ <= 50) {
        if (amrmTokenSecretManagerForRM1.getCurrnetMasterKeyData().getMasterKey().getKeyId() != token.decodeIdentifier().getKeyId()) {
            break;
        }
        try {
            amClient.allocate(0.1f);
        } catch (Exception ex) {
            break;
        }
        Thread.sleep(500);
    }
    Assert.assertTrue(amrmTokenSecretManagerForRM1.getNextMasterKeyData() == null);
    Assert.assertTrue(amrmTokenSecretManagerForRM1.getCurrnetMasterKeyData().getMasterKey().getKeyId() == newToken.decodeIdentifier().getKeyId());
    // start 2nd RM
    conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, "0.0.0.0:" + ServerSocketUtil.getPort(45020, 10));
    final MyResourceManager2 rm2 = new MyResourceManager2(conf, memStore);
    rm2.start();
    nm1.setResourceTrackerService(rm2.getResourceTrackerService());
    ((MyAMRMClientImpl) amClient).updateRMProxy(rm2);
    dispatcher = (DrainDispatcher) rm2.getRMContext().getDispatcher();
    AMRMTokenSecretManager amrmTokenSecretManagerForRM2 = rm2.getRMContext().getAMRMTokenSecretManager();
    Assert.assertTrue(amrmTokenSecretManagerForRM2.getCurrnetMasterKeyData().getMasterKey().getKeyId() == newToken.decodeIdentifier().getKeyId());
    Assert.assertTrue(amrmTokenSecretManagerForRM2.getNextMasterKeyData() == null);
    try {
        UserGroupInformation testUser = UserGroupInformation.createRemoteUser("testUser");
        SecurityUtil.setTokenService(token, rm2.getApplicationMasterService().getBindAddress());
        testUser.addToken(token);
        testUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {

            @Override
            public ApplicationMasterProtocol run() {
                return (ApplicationMasterProtocol) YarnRPC.create(conf).getProxy(ApplicationMasterProtocol.class, rm2.getApplicationMasterService().getBindAddress(), conf);
            }
        }).allocate(Records.newRecord(AllocateRequest.class));
        Assert.fail("The old Token should not work");
    } catch (Exception ex) {
        Assert.assertTrue(ex instanceof InvalidToken);
        Assert.assertTrue(ex.getMessage().contains("Invalid AMRMToken from " + token.decodeIdentifier().getApplicationAttemptId()));
    }
    // make sure the recovered AMRMToken works for new RM
    amClient.allocate(0.1f);
    amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null);
    amClient.stop();
    rm1.stop();
    rm2.stop();
}
Also used : DrainDispatcher(org.apache.hadoop.yarn.event.DrainDispatcher) RMApp(org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp) MockNM(org.apache.hadoop.yarn.server.resourcemanager.MockNM) AllocateRequest(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest) ApplicationMasterProtocol(org.apache.hadoop.yarn.api.ApplicationMasterProtocol) ApplicationAttemptId(org.apache.hadoop.yarn.api.records.ApplicationAttemptId) AMRMTokenSecretManager(org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager) IOException(java.io.IOException) MemoryRMStateStore(org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore) AMRMTokenIdentifier(org.apache.hadoop.yarn.security.AMRMTokenIdentifier) PrivilegedAction(java.security.PrivilegedAction) InvalidToken(org.apache.hadoop.security.token.SecretManager.InvalidToken) ContainerRequest(org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest) UpdateContainerRequest(org.apache.hadoop.yarn.api.records.UpdateContainerRequest) UserGroupInformation(org.apache.hadoop.security.UserGroupInformation) Test(org.junit.Test)

Example 9 with ApplicationMasterProtocol

use of org.apache.hadoop.yarn.api.ApplicationMasterProtocol in project hadoop by apache.

the class TestDistributedScheduling method testAMRMClient.

/**
   * Validates if AMRMClient can be used with Distributed Scheduling turned on.
   *
   * @throws Exception
   */
@Test(timeout = 120000)
@SuppressWarnings("unchecked")
public void testAMRMClient() throws Exception {
    AMRMClientImpl<AMRMClient.ContainerRequest> amClient = null;
    try {
        Priority priority = Priority.newInstance(1);
        Priority priority2 = Priority.newInstance(2);
        Resource capability = Resource.newInstance(1024, 1);
        List<NodeReport> nodeReports = rmClient.getNodeReports(NodeState.RUNNING);
        String node = nodeReports.get(0).getNodeId().getHost();
        String rack = nodeReports.get(0).getRackName();
        String[] nodes = new String[] { node };
        String[] racks = new String[] { rack };
        // start am rm client
        amClient = new AMRMClientImpl(client);
        amClient.init(yarnConf);
        amClient.start();
        amClient.registerApplicationMaster(NetUtils.getHostname(), 1024, "");
        assertEquals(0, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
        RemoteRequestsTable<ContainerRequest> remoteRequestsTable = amClient.getTable(0);
        int containersRequestedNode = remoteRequestsTable.get(priority, node, ExecutionType.GUARANTEED, capability).remoteRequest.getNumContainers();
        int containersRequestedRack = remoteRequestsTable.get(priority, rack, ExecutionType.GUARANTEED, capability).remoteRequest.getNumContainers();
        int containersRequestedAny = remoteRequestsTable.get(priority, ResourceRequest.ANY, ExecutionType.GUARANTEED, capability).remoteRequest.getNumContainers();
        int oppContainersRequestedAny = remoteRequestsTable.get(priority2, ResourceRequest.ANY, ExecutionType.OPPORTUNISTIC, capability).remoteRequest.getNumContainers();
        assertEquals(2, containersRequestedNode);
        assertEquals(2, containersRequestedRack);
        assertEquals(2, containersRequestedAny);
        assertEquals(1, oppContainersRequestedAny);
        assertEquals(4, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        // RM should allocate container within 2 calls to allocate()
        int allocatedContainerCount = 0;
        int iterationsLeft = 10;
        Set<ContainerId> releases = new TreeSet<>();
        amClient.getNMTokenCache().clearCache();
        Assert.assertEquals(0, amClient.getNMTokenCache().numberOfTokensInCache());
        HashMap<String, Token> receivedNMTokens = new HashMap<>();
        while (allocatedContainerCount < (containersRequestedAny + oppContainersRequestedAny) && iterationsLeft-- > 0) {
            AllocateResponse allocResponse = amClient.allocate(0.1f);
            assertEquals(0, amClient.ask.size());
            assertEquals(0, amClient.release.size());
            allocatedContainerCount += allocResponse.getAllocatedContainers().size();
            for (Container container : allocResponse.getAllocatedContainers()) {
                ContainerId rejectContainerId = container.getId();
                releases.add(rejectContainerId);
            }
            for (NMToken token : allocResponse.getNMTokens()) {
                String nodeID = token.getNodeId().toString();
                receivedNMTokens.put(nodeID, token.getToken());
            }
            if (allocatedContainerCount < containersRequestedAny) {
                // sleep to let NM's heartbeat to RM and trigger allocations
                sleep(100);
            }
        }
        assertEquals(allocatedContainerCount, containersRequestedAny + oppContainersRequestedAny);
        for (ContainerId rejectContainerId : releases) {
            amClient.releaseAssignedContainer(rejectContainerId);
        }
        assertEquals(3, amClient.release.size());
        assertEquals(0, amClient.ask.size());
        // need to tell the AMRMClient that we dont need these resources anymore
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
        assertEquals(4, amClient.ask.size());
        // test RPC exception handling
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
        amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
        final AMRMClient amc = amClient;
        ApplicationMasterProtocol realRM = amClient.rmClient;
        try {
            ApplicationMasterProtocol mockRM = mock(ApplicationMasterProtocol.class);
            when(mockRM.allocate(any(AllocateRequest.class))).thenAnswer(new Answer<AllocateResponse>() {

                public AllocateResponse answer(InvocationOnMock invocation) throws Exception {
                    amc.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
                    amc.removeContainerRequest(new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
                    amc.removeContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority2, 0, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true)));
                    throw new Exception();
                }
            });
            amClient.rmClient = mockRM;
            amClient.allocate(0.1f);
        } catch (Exception ioe) {
        } finally {
            amClient.rmClient = realRM;
        }
        assertEquals(3, amClient.release.size());
        assertEquals(6, amClient.ask.size());
        iterationsLeft = 3;
        // do a few iterations to ensure RM is not going send new containers
        while (iterationsLeft-- > 0) {
            // inform RM of rejection
            AllocateResponse allocResponse = amClient.allocate(0.1f);
            // RM did not send new containers because AM does not need any
            assertEquals(0, allocResponse.getAllocatedContainers().size());
            if (allocResponse.getCompletedContainersStatuses().size() > 0) {
                for (ContainerStatus cStatus : allocResponse.getCompletedContainersStatuses()) {
                    if (releases.contains(cStatus.getContainerId())) {
                        assertEquals(cStatus.getState(), ContainerState.COMPLETE);
                        assertEquals(-100, cStatus.getExitStatus());
                        releases.remove(cStatus.getContainerId());
                    }
                }
            }
            if (iterationsLeft > 0) {
                // sleep to make sure NM's heartbeat
                sleep(100);
            }
        }
        assertEquals(0, amClient.ask.size());
        assertEquals(0, amClient.release.size());
        amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null);
    } finally {
        if (amClient != null && amClient.getServiceState() == Service.STATE.STARTED) {
            amClient.stop();
        }
    }
}
Also used : AMRMClient(org.apache.hadoop.yarn.client.api.AMRMClient) HashMap(java.util.HashMap) AllocateRequest(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest) NMToken(org.apache.hadoop.yarn.api.records.NMToken) Token(org.apache.hadoop.yarn.api.records.Token) ApplicationMasterProtocol(org.apache.hadoop.yarn.api.ApplicationMasterProtocol) AllocateResponse(org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse) RMContainer(org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer) Container(org.apache.hadoop.yarn.api.records.Container) ContainerStatus(org.apache.hadoop.yarn.api.records.ContainerStatus) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) TreeSet(java.util.TreeSet) ContainerRequest(org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest) ContainerRequest(org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest) NMToken(org.apache.hadoop.yarn.api.records.NMToken) Priority(org.apache.hadoop.yarn.api.records.Priority) Resource(org.apache.hadoop.yarn.api.records.Resource) InvocationOnMock(org.mockito.invocation.InvocationOnMock) NodeReport(org.apache.hadoop.yarn.api.records.NodeReport) Test(org.junit.Test)

Example 10 with ApplicationMasterProtocol

use of org.apache.hadoop.yarn.api.ApplicationMasterProtocol in project hadoop by apache.

the class TestUnmanagedAMLauncher method main.

// provide main method so this class can act as AM
public static void main(String[] args) throws Exception {
    if (args[0].equals("success")) {
        ApplicationMasterProtocol client = ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class);
        client.registerApplicationMaster(RegisterApplicationMasterRequest.newInstance(NetUtils.getHostname(), -1, ""));
        Thread.sleep(1000);
        FinishApplicationMasterResponse resp = client.finishApplicationMaster(FinishApplicationMasterRequest.newInstance(FinalApplicationStatus.SUCCEEDED, "success", null));
        assertTrue(resp.getIsUnregistered());
        System.exit(0);
    } else {
        System.exit(1);
    }
}
Also used : ApplicationMasterProtocol(org.apache.hadoop.yarn.api.ApplicationMasterProtocol) FinishApplicationMasterResponse(org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse)

Aggregations

ApplicationMasterProtocol (org.apache.hadoop.yarn.api.ApplicationMasterProtocol)20 Test (org.junit.Test)15 Configuration (org.apache.hadoop.conf.Configuration)12 AllocateRequest (org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)12 ApplicationAttemptId (org.apache.hadoop.yarn.api.records.ApplicationAttemptId)10 IOException (java.io.IOException)9 UserGroupInformation (org.apache.hadoop.security.UserGroupInformation)9 YarnConfiguration (org.apache.hadoop.yarn.conf.YarnConfiguration)9 InetSocketAddress (java.net.InetSocketAddress)8 RMApp (org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp)8 RegisterApplicationMasterRequest (org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest)7 AllocateResponse (org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse)6 YarnRPC (org.apache.hadoop.yarn.ipc.YarnRPC)6 RMAppAttempt (org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt)6 HashMap (java.util.HashMap)5 Credentials (org.apache.hadoop.security.Credentials)5 MockNM (org.apache.hadoop.yarn.server.resourcemanager.MockNM)5 ContainerRequest (org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest)4 YarnException (org.apache.hadoop.yarn.exceptions.YarnException)4 AMRMTokenIdentifier (org.apache.hadoop.yarn.security.AMRMTokenIdentifier)4