Examples with RMAppAttempt org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt used on opensource projects

Example 21 with RMAppAttempt

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt in project hadoop by apache.

the class TestApplicationMasterService method testInvalidIncreaseDecreaseRequest.

@Test(timeout = 60000)
public void testInvalidIncreaseDecreaseRequest() throws Exception {
    conf = new YarnConfiguration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    MockRM rm = new MockRM(conf);
    try {
        rm.start();
        // Register node1
        MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
        // Submit an application
        RMApp app1 = rm.submitApp(1024);
        // kick the scheduling
        nm1.nodeHeartbeat(true);
        RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
        MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
        RegisterApplicationMasterResponse registerResponse = am1.registerAppAttempt();
        sentRMContainerLaunched(rm, ContainerId.newContainerId(am1.getApplicationAttemptId(), 1));
        // Ask for a normal increase should be successfull
        am1.sendContainerResizingRequest(Arrays.asList(UpdateContainerRequest.newInstance(0, ContainerId.newContainerId(attempt1.getAppAttemptId(), 1), ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(2048), null)));
        // Target resource is negative, should fail
        AllocateResponse response = am1.sendContainerResizingRequest(Arrays.asList(UpdateContainerRequest.newInstance(0, ContainerId.newContainerId(attempt1.getAppAttemptId(), 1), ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(-1), null)));
        Assert.assertEquals(1, response.getUpdateErrors().size());
        Assert.assertEquals("RESOURCE_OUTSIDE_ALLOWED_RANGE", response.getUpdateErrors().get(0).getReason());
        // Target resource is more than maxAllocation, should fail
        response = am1.sendContainerResizingRequest(Arrays.asList(UpdateContainerRequest.newInstance(0, ContainerId.newContainerId(attempt1.getAppAttemptId(), 1), ContainerUpdateType.INCREASE_RESOURCE, Resources.add(registerResponse.getMaximumResourceCapability(), Resources.createResource(1)), null)));
        Assert.assertEquals(1, response.getUpdateErrors().size());
        Assert.assertEquals("RESOURCE_OUTSIDE_ALLOWED_RANGE", response.getUpdateErrors().get(0).getReason());
        // Contains multiple increase/decrease requests for same contaienrId 
        response = am1.sendContainerResizingRequest(Arrays.asList(UpdateContainerRequest.newInstance(0, ContainerId.newContainerId(attempt1.getAppAttemptId(), 1), ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(2048, 4), null), UpdateContainerRequest.newInstance(0, ContainerId.newContainerId(attempt1.getAppAttemptId(), 1), ContainerUpdateType.DECREASE_RESOURCE, Resources.createResource(1024, 1), null)));
        Assert.assertEquals(1, response.getUpdateErrors().size());
        Assert.assertEquals("UPDATE_OUTSTANDING_ERROR", response.getUpdateErrors().get(0).getReason());
    } finally {
        if (rm != null) {
            rm.close();
        }
    }
}

Example 22 with RMAppAttempt

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt in project hadoop by apache.

the class TestApplicationMasterService method testRMIdentifierOnContainerAllocation.

@Test(timeout = 3000000)
public void testRMIdentifierOnContainerAllocation() throws Exception {
    MockRM rm = new MockRM(conf);
    rm.start();
    // Register node1
    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
    // Submit an application
    RMApp app1 = rm.submitApp(2048);
    // kick the scheduling
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
    MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
    am1.registerAppAttempt();
    am1.addRequests(new String[] { "127.0.0.1" }, GB, 1, 1);
    // send the request
    AllocateResponse alloc1Response = am1.schedule();
    // kick the scheduler
    nm1.nodeHeartbeat(true);
    while (alloc1Response.getAllocatedContainers().size() < 1) {
        LOG.info("Waiting for containers to be created for app 1...");
        sleep(1000);
        alloc1Response = am1.schedule();
    }
    // assert RMIdentifer is set properly in allocated containers
    Container allocatedContainer = alloc1Response.getAllocatedContainers().get(0);
    ContainerTokenIdentifier tokenId = BuilderUtils.newContainerTokenIdentifier(allocatedContainer.getContainerToken());
    Assert.assertEquals(MockRM.getClusterTimeStamp(), tokenId.getRMIdentifier());
    rm.stop();
}

Example 23 with RMAppAttempt

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt in project hadoop by apache.

the class TestApplicationMasterService method testInvalidContainerReleaseRequest.

@Test(timeout = 600000)
public void testInvalidContainerReleaseRequest() throws Exception {
    MockRM rm = new MockRM(conf);
    try {
        rm.start();
        // Register node1
        MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
        // Submit an application
        RMApp app1 = rm.submitApp(1024);
        // kick the scheduling
        nm1.nodeHeartbeat(true);
        RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
        MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
        am1.registerAppAttempt();
        am1.addRequests(new String[] { "127.0.0.1" }, GB, 1, 1);
        // send the request
        AllocateResponse alloc1Response = am1.schedule();
        // kick the scheduler
        nm1.nodeHeartbeat(true);
        while (alloc1Response.getAllocatedContainers().size() < 1) {
            LOG.info("Waiting for containers to be created for app 1...");
            sleep(1000);
            alloc1Response = am1.schedule();
        }
        Assert.assertTrue(alloc1Response.getAllocatedContainers().size() > 0);
        RMApp app2 = rm.submitApp(1024);
        nm1.nodeHeartbeat(true);
        RMAppAttempt attempt2 = app2.getCurrentAppAttempt();
        MockAM am2 = rm.sendAMLaunched(attempt2.getAppAttemptId());
        am2.registerAppAttempt();
        // Now trying to release container allocated for app1 -> appAttempt1.
        ContainerId cId = alloc1Response.getAllocatedContainers().get(0).getId();
        am2.addContainerToBeReleased(cId);
        try {
            am2.schedule();
            Assert.fail("Exception was expected!!");
        } catch (InvalidContainerReleaseException e) {
            StringBuilder sb = new StringBuilder("Cannot release container : ");
            sb.append(cId.toString());
            sb.append(" not belonging to this application attempt : ");
            sb.append(attempt2.getAppAttemptId().toString());
            Assert.assertTrue(e.getMessage().contains(sb.toString()));
        }
    } finally {
        if (rm != null) {
            rm.stop();
        }
    }
}

Example 24 with RMAppAttempt

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt in project hadoop by apache.

the class TestContainerResourceUsage method testUsageWithOneAttemptAndOneContainer.

@Test(timeout = 120000)
public void testUsageWithOneAttemptAndOneContainer() throws Exception {
    MockRM rm = new MockRM(conf);
    rm.start();
    MockNM nm = new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
    nm.registerNode();
    RMApp app0 = rm.submitApp(200);
    RMAppMetrics rmAppMetrics = app0.getRMAppMetrics();
    Assert.assertTrue("Before app submittion, memory seconds should have been 0 but was " + rmAppMetrics.getMemorySeconds(), rmAppMetrics.getMemorySeconds() == 0);
    Assert.assertTrue("Before app submission, vcore seconds should have been 0 but was " + rmAppMetrics.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds() == 0);
    RMAppAttempt attempt0 = app0.getCurrentAppAttempt();
    nm.nodeHeartbeat(true);
    MockAM am0 = rm.sendAMLaunched(attempt0.getAppAttemptId());
    am0.registerAppAttempt();
    RMContainer rmContainer = rm.getResourceScheduler().getRMContainer(attempt0.getMasterContainer().getId());
    // Allow metrics to accumulate.
    int sleepInterval = 1000;
    int cumulativeSleepTime = 0;
    while (rmAppMetrics.getMemorySeconds() <= 0 && cumulativeSleepTime < 5000) {
        Thread.sleep(sleepInterval);
        cumulativeSleepTime += sleepInterval;
    }
    rmAppMetrics = app0.getRMAppMetrics();
    Assert.assertTrue("While app is running, memory seconds should be >0 but is " + rmAppMetrics.getMemorySeconds(), rmAppMetrics.getMemorySeconds() > 0);
    Assert.assertTrue("While app is running, vcore seconds should be >0 but is " + rmAppMetrics.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds() > 0);
    MockRM.finishAMAndVerifyAppState(app0, rm, nm, am0);
    AggregateAppResourceUsage ru = calculateContainerResourceMetrics(rmContainer);
    rmAppMetrics = app0.getRMAppMetrics();
    Assert.assertEquals("Unexpected MemorySeconds value", ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds());
    Assert.assertEquals("Unexpected VcoreSeconds value", ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds());
    rm.stop();
}

Example 25 with RMAppAttempt

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt in project hadoop by apache.

the class TestAMAuthorization method testUnauthorizedAccess.

@Test
public void testUnauthorizedAccess() throws Exception {
    MyContainerManager containerManager = new MyContainerManager();
    rm = new MockRMWithAMS(conf, containerManager);
    rm.start();
    MockNM nm1 = rm.registerNode("localhost:1234", 5120);
    RMApp app = rm.submitApp(1024);
    nm1.nodeHeartbeat(true);
    int waitCount = 0;
    while (containerManager.containerTokens == null && waitCount++ < 40) {
        LOG.info("Waiting for AM Launch to happen..");
        Thread.sleep(1000);
    }
    Assert.assertNotNull(containerManager.containerTokens);
    RMAppAttempt attempt = app.getCurrentAppAttempt();
    ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
    waitForLaunchedState(attempt);
    final Configuration conf = rm.getConfig();
    final YarnRPC rpc = YarnRPC.create(conf);
    final InetSocketAddress serviceAddr = conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
    UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(applicationAttemptId.toString());
    // First try contacting NM without tokens
    ApplicationMasterProtocol client = currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {

        @Override
        public ApplicationMasterProtocol run() {
            return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class, serviceAddr, conf);
        }
    });
    RegisterApplicationMasterRequest request = Records.newRecord(RegisterApplicationMasterRequest.class);
    try {
        client.registerApplicationMaster(request);
        Assert.fail("Should fail with authorization error");
    } catch (Exception e) {
        if (isCause(AccessControlException.class, e)) {
            // Because there are no tokens, the request should be rejected as the
            // server side will assume we are trying simple auth.
            String expectedMessage = "";
            if (UserGroupInformation.isSecurityEnabled()) {
                expectedMessage = "Client cannot authenticate via:[TOKEN]";
            } else {
                expectedMessage = "SIMPLE authentication is not enabled.  Available:[TOKEN]";
            }
            Assert.assertTrue(e.getCause().getMessage().contains(expectedMessage));
        } else {
            throw e;
        }
    }
// TODO: Add validation of invalid authorization when there's more data in
// the AMRMToken
}