Examples with MockAM org.apache.hadoop.yarn.server.resourcemanager.MockAM used on opensource projects

Example 61 with MockAM

use of org.apache.hadoop.yarn.server.resourcemanager.MockAM in project hadoop by apache.

the class TestCapacityScheduler method testRecoverRequestAfterPreemption.

@Test(timeout = 300000)
public void testRecoverRequestAfterPreemption() throws Exception {
    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    MockRM rm1 = new MockRM(conf);
    rm1.start();
    MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000);
    RMApp app1 = rm1.submitApp(1024);
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
    CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
    // request a container.
    am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
    ContainerId containerId1 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
    rm1.waitForState(nm1, containerId1, RMContainerState.ALLOCATED);
    RMContainer rmContainer = cs.getRMContainer(containerId1);
    List<ResourceRequest> requests = rmContainer.getResourceRequests();
    FiCaSchedulerApp app = cs.getApplicationAttempt(am1.getApplicationAttemptId());
    FiCaSchedulerNode node = cs.getNode(rmContainer.getAllocatedNode());
    for (ResourceRequest request : requests) {
        // Skip the OffRack and RackLocal resource requests.
        if (request.getResourceName().equals(node.getRackName()) || request.getResourceName().equals(ResourceRequest.ANY)) {
            continue;
        }
        // Already the node local resource request is cleared from RM after
        // allocation.
        Assert.assertEquals(0, app.getOutstandingAsksCount(SchedulerRequestKey.create(request), request.getResourceName()));
    }
    // Call killContainer to preempt the container
    cs.markContainerForKillable(rmContainer);
    Assert.assertEquals(3, requests.size());
    for (ResourceRequest request : requests) {
        // Resource request must have added back in RM after preempt event
        // handling.
        Assert.assertEquals(1, app.getOutstandingAsksCount(SchedulerRequestKey.create(request), request.getResourceName()));
    }
    // New container will be allocated and will move to ALLOCATED state
    ContainerId containerId2 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 3);
    rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);
    // allocate container
    List<Container> containers = am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers();
    // Now with updated ResourceRequest, a container is allocated for AM.
    Assert.assertTrue(containers.size() == 1);
}

Example 62 with MockAM

use of org.apache.hadoop.yarn.server.resourcemanager.MockAM in project hadoop by apache.

the class TestCapacityScheduler method testPendingResourceUpdatedAccordingToIncreaseRequestChanges.

@Test
public void testPendingResourceUpdatedAccordingToIncreaseRequestChanges() throws Exception {
    Configuration conf = TestUtils.getConfigurationWithQueueLabels(new Configuration(false));
    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
    final RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
    mgr.init(conf);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    MockRM rm = new MockRM(conf, memStore) {

        protected RMNodeLabelsManager createNodeLabelManager() {
            return mgr;
        }
    };
    rm.start();
    // label = ""
    MockNM nm1 = new MockNM("h1:1234", 200 * GB, rm.getResourceTrackerService());
    nm1.registerNode();
    // Launch app1 in queue=a1
    RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a1");
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
    // Allocate two more containers
    am1.allocate(Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1), "*", Resources.createResource(2 * GB), 2)), null);
    ContainerId containerId1 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
    ContainerId containerId2 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
    ContainerId containerId3 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 3);
    Assert.assertTrue(rm.waitForState(nm1, containerId3, RMContainerState.ALLOCATED));
    // Acquire them
    am1.allocate(null, null);
    sentRMContainerLaunched(rm, ContainerId.newContainerId(am1.getApplicationAttemptId(), 1L));
    sentRMContainerLaunched(rm, ContainerId.newContainerId(am1.getApplicationAttemptId(), 2L));
    sentRMContainerLaunched(rm, ContainerId.newContainerId(am1.getApplicationAttemptId(), 3L));
    // am1 asks to change its AM container from 1GB to 3GB
    am1.sendContainerResizingRequest(Arrays.asList(UpdateContainerRequest.newInstance(0, containerId1, ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(3 * GB), null)));
    FiCaSchedulerApp app = getFiCaSchedulerApp(rm, app1.getApplicationId());
    Assert.assertEquals(2 * GB, app.getAppAttemptResourceUsage().getPending().getMemorySize());
    checkPendingResource(rm, "a1", 2 * GB, null);
    checkPendingResource(rm, "a", 2 * GB, null);
    checkPendingResource(rm, "root", 2 * GB, null);
    // am1 asks to change containerId2 (2G -> 3G) and containerId3 (2G -> 5G)
    am1.sendContainerResizingRequest(Arrays.asList(UpdateContainerRequest.newInstance(0, containerId2, ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(3 * GB), null), UpdateContainerRequest.newInstance(0, containerId3, ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(5 * GB), null)));
    Assert.assertEquals(6 * GB, app.getAppAttemptResourceUsage().getPending().getMemorySize());
    checkPendingResource(rm, "a1", 6 * GB, null);
    checkPendingResource(rm, "a", 6 * GB, null);
    checkPendingResource(rm, "root", 6 * GB, null);
    // am1 asks to change containerId1 (1G->3G), containerId2 (2G -> 4G) and
    // containerId3 (2G -> 2G)
    am1.sendContainerResizingRequest(Arrays.asList(UpdateContainerRequest.newInstance(0, containerId1, ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(3 * GB), null), UpdateContainerRequest.newInstance(0, containerId2, ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(4 * GB), null), UpdateContainerRequest.newInstance(0, containerId3, ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(2 * GB), null)));
    Assert.assertEquals(4 * GB, app.getAppAttemptResourceUsage().getPending().getMemorySize());
    checkPendingResource(rm, "a1", 4 * GB, null);
    checkPendingResource(rm, "a", 4 * GB, null);
    checkPendingResource(rm, "root", 4 * GB, null);
}

Example 63 with MockAM

use of org.apache.hadoop.yarn.server.resourcemanager.MockAM in project hadoop by apache.

the class TestRMWebServicesApps method testAppsQueryFinishEnd.

@Test
public void testAppsQueryFinishEnd() throws JSONException, Exception {
    rm.start();
    MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
    RMApp app1 = rm.submitApp(CONTAINER_MB);
    amNodeManager.nodeHeartbeat(true);
    // finish App
    MockAM am = rm.sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId());
    am.registerAppAttempt();
    am.unregisterAppAttempt();
    amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(), 1, ContainerState.COMPLETE);
    rm.submitApp(CONTAINER_MB);
    rm.submitApp(CONTAINER_MB);
    long end = System.currentTimeMillis();
    WebResource r = resource();
    ClientResponse response = r.path("ws").path("v1").path("cluster").path("apps").queryParam("finishedTimeEnd", String.valueOf(end)).accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject apps = json.getJSONObject("apps");
    assertEquals("incorrect number of elements", 1, apps.length());
    JSONArray array = apps.getJSONArray("app");
    assertEquals("incorrect number of elements", 3, array.length());
    rm.stop();
}

Example 64 with MockAM

use of org.apache.hadoop.yarn.server.resourcemanager.MockAM in project hadoop by apache.

the class TestRMWebServicesApps method testAppsQueryAppTypes.

@Test
public void testAppsQueryAppTypes() throws JSONException, Exception {
    rm.start();
    MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
    Thread.sleep(1);
    RMApp app1 = rm.submitApp(CONTAINER_MB);
    amNodeManager.nodeHeartbeat(true);
    // finish App
    MockAM am = rm.sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId());
    am.registerAppAttempt();
    am.unregisterAppAttempt();
    amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(), 1, ContainerState.COMPLETE);
    rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser().getShortUserName(), null, false, null, 2, null, "MAPREDUCE");
    rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser().getShortUserName(), null, false, null, 2, null, "NON-YARN");
    WebResource r = resource();
    ClientResponse response = r.path("ws").path("v1").path("cluster").path("apps").queryParam("applicationTypes", "MAPREDUCE").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
    JSONObject json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    JSONObject apps = json.getJSONObject("apps");
    assertEquals("incorrect number of elements", 1, apps.length());
    JSONArray array = apps.getJSONArray("app");
    assertEquals("incorrect number of elements", 1, array.length());
    assertEquals("MAPREDUCE", array.getJSONObject(0).getString("applicationType"));
    r = resource();
    response = r.path("ws").path("v1").path("cluster").path("apps").queryParam("applicationTypes", "YARN").queryParam("applicationTypes", "MAPREDUCE").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
    json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    apps = json.getJSONObject("apps");
    assertEquals("incorrect number of elements", 1, apps.length());
    array = apps.getJSONArray("app");
    assertEquals("incorrect number of elements", 2, array.length());
    assertTrue((array.getJSONObject(0).getString("applicationType").equals("YARN") && array.getJSONObject(1).getString("applicationType").equals("MAPREDUCE")) || (array.getJSONObject(1).getString("applicationType").equals("YARN") && array.getJSONObject(0).getString("applicationType").equals("MAPREDUCE")));
    r = resource();
    response = r.path("ws").path("v1").path("cluster").path("apps").queryParam("applicationTypes", "YARN,NON-YARN").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
    json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    apps = json.getJSONObject("apps");
    assertEquals("incorrect number of elements", 1, apps.length());
    array = apps.getJSONArray("app");
    assertEquals("incorrect number of elements", 2, array.length());
    assertTrue((array.getJSONObject(0).getString("applicationType").equals("YARN") && array.getJSONObject(1).getString("applicationType").equals("NON-YARN")) || (array.getJSONObject(1).getString("applicationType").equals("YARN") && array.getJSONObject(0).getString("applicationType").equals("NON-YARN")));
    r = resource();
    response = r.path("ws").path("v1").path("cluster").path("apps").queryParam("applicationTypes", "").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
    json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    apps = json.getJSONObject("apps");
    assertEquals("incorrect number of elements", 1, apps.length());
    array = apps.getJSONArray("app");
    assertEquals("incorrect number of elements", 3, array.length());
    r = resource();
    response = r.path("ws").path("v1").path("cluster").path("apps").queryParam("applicationTypes", "YARN,NON-YARN").queryParam("applicationTypes", "MAPREDUCE").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
    json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    apps = json.getJSONObject("apps");
    assertEquals("incorrect number of elements", 1, apps.length());
    array = apps.getJSONArray("app");
    assertEquals("incorrect number of elements", 3, array.length());
    r = resource();
    response = r.path("ws").path("v1").path("cluster").path("apps").queryParam("applicationTypes", "YARN").queryParam("applicationTypes", "").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
    json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    apps = json.getJSONObject("apps");
    assertEquals("incorrect number of elements", 1, apps.length());
    array = apps.getJSONArray("app");
    assertEquals("incorrect number of elements", 1, array.length());
    assertEquals("YARN", array.getJSONObject(0).getString("applicationType"));
    r = resource();
    response = r.path("ws").path("v1").path("cluster").path("apps").queryParam("applicationTypes", ",,, ,, YARN ,, ,").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
    json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    apps = json.getJSONObject("apps");
    assertEquals("incorrect number of elements", 1, apps.length());
    array = apps.getJSONArray("app");
    assertEquals("incorrect number of elements", 1, array.length());
    assertEquals("YARN", array.getJSONObject(0).getString("applicationType"));
    r = resource();
    response = r.path("ws").path("v1").path("cluster").path("apps").queryParam("applicationTypes", ",,, ,,  ,, ,").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
    json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    apps = json.getJSONObject("apps");
    assertEquals("incorrect number of elements", 1, apps.length());
    array = apps.getJSONArray("app");
    assertEquals("incorrect number of elements", 3, array.length());
    r = resource();
    response = r.path("ws").path("v1").path("cluster").path("apps").queryParam("applicationTypes", "YARN, ,NON-YARN, ,,").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
    json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    apps = json.getJSONObject("apps");
    assertEquals("incorrect number of elements", 1, apps.length());
    array = apps.getJSONArray("app");
    assertEquals("incorrect number of elements", 2, array.length());
    assertTrue((array.getJSONObject(0).getString("applicationType").equals("YARN") && array.getJSONObject(1).getString("applicationType").equals("NON-YARN")) || (array.getJSONObject(1).getString("applicationType").equals("YARN") && array.getJSONObject(0).getString("applicationType").equals("NON-YARN")));
    r = resource();
    response = r.path("ws").path("v1").path("cluster").path("apps").queryParam("applicationTypes", " YARN, ,  ,,,").queryParam("applicationTypes", "MAPREDUCE , ,, ,").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
    json = response.getEntity(JSONObject.class);
    assertEquals("incorrect number of elements", 1, json.length());
    apps = json.getJSONObject("apps");
    assertEquals("incorrect number of elements", 1, apps.length());
    array = apps.getJSONArray("app");
    assertEquals("incorrect number of elements", 2, array.length());
    assertTrue((array.getJSONObject(0).getString("applicationType").equals("YARN") && array.getJSONObject(1).getString("applicationType").equals("MAPREDUCE")) || (array.getJSONObject(1).getString("applicationType").equals("YARN") && array.getJSONObject(0).getString("applicationType").equals("MAPREDUCE")));
    rm.stop();
}

Example 65 with MockAM

use of org.apache.hadoop.yarn.server.resourcemanager.MockAM in project hadoop by apache.

the class TestRMWebServicesApps method testMultipleAppAttempts.

@Test(timeout = 20000)
public void testMultipleAppAttempts() throws JSONException, Exception {
    rm.start();
    MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 8192);
    RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
    MockAM am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
    int maxAppAttempts = rm.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
    assertTrue(maxAppAttempts > 1);
    int numAttempt = 1;
    while (true) {
        // fail the AM by sending CONTAINER_FINISHED event without registering.
        amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
        rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FAILED);
        if (numAttempt == maxAppAttempts) {
            rm.waitForState(app1.getApplicationId(), RMAppState.FAILED);
            break;
        }
        // wait for app to start a new attempt.
        rm.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
        am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
        numAttempt++;
    }
    assertEquals("incorrect number of attempts", maxAppAttempts, app1.getAppAttempts().values().size());
    testAppAttemptsHelper(app1.getApplicationId().toString(), app1, MediaType.APPLICATION_JSON);
    rm.stop();
}