Examples with AllocateRequest org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest used on opensource projects

Example 21 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class MRAppBenchmark method benchmark1.

@Test
public void benchmark1() throws Exception {
    // Adjust for benchmarking. Start with thousands.
    int maps = 100;
    int reduces = 0;
    System.out.println("Running benchmark with maps:" + maps + " reduces:" + reduces);
    run(new MRApp(maps, reduces, true, this.getClass().getName(), true) {

        @Override
        protected ContainerAllocator createContainerAllocator(ClientService clientService, AppContext context) {
            AMPreemptionPolicy policy = new NoopAMPreemptionPolicy();
            return new RMContainerAllocator(clientService, context, policy) {

                @Override
                protected ApplicationMasterProtocol createSchedulerProxy() {
                    return new ApplicationMasterProtocol() {

                        @Override
                        public RegisterApplicationMasterResponse registerApplicationMaster(RegisterApplicationMasterRequest request) throws IOException {
                            RegisterApplicationMasterResponse response = Records.newRecord(RegisterApplicationMasterResponse.class);
                            response.setMaximumResourceCapability(Resource.newInstance(10240, 1));
                            return response;
                        }

                        @Override
                        public FinishApplicationMasterResponse finishApplicationMaster(FinishApplicationMasterRequest request) throws IOException {
                            FinishApplicationMasterResponse response = Records.newRecord(FinishApplicationMasterResponse.class);
                            return response;
                        }

                        @Override
                        public AllocateResponse allocate(AllocateRequest request) throws IOException {
                            AllocateResponse response = Records.newRecord(AllocateResponse.class);
                            List<ResourceRequest> askList = request.getAskList();
                            List<Container> containers = new ArrayList<Container>();
                            for (ResourceRequest req : askList) {
                                if (!ResourceRequest.isAnyLocation(req.getResourceName())) {
                                    continue;
                                }
                                int numContainers = req.getNumContainers();
                                for (int i = 0; i < numContainers; i++) {
                                    ContainerId containerId = ContainerId.newContainerId(getContext().getApplicationAttemptId(), request.getResponseId() + i);
                                    containers.add(Container.newInstance(containerId, NodeId.newInstance("host" + containerId.getContainerId(), 2345), "host" + containerId.getContainerId() + ":5678", req.getCapability(), req.getPriority(), null));
                                }
                            }
                            response.setAllocatedContainers(containers);
                            response.setResponseId(request.getResponseId() + 1);
                            response.setNumClusterNodes(350);
                            return response;
                        }
                    };
                }
            };
        }
    });
}

Example 22 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class TestAMRMProxy method testAMRMProxyE2E.

/*
   * This test validates register, allocate and finish of an application through
   * the AMRMPRoxy.
   */
@Test(timeout = 120000)
public void testAMRMProxyE2E() throws Exception {
    ApplicationMasterProtocol client;
    try (MiniYARNCluster cluster = new MiniYARNCluster("testAMRMProxyE2E", 1, 1, 1);
        YarnClient rmClient = YarnClient.createYarnClient()) {
        Configuration conf = new YarnConfiguration();
        conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true);
        cluster.init(conf);
        cluster.start();
        final Configuration yarnConf = cluster.getConfig();
        // the client has to connect to AMRMProxy
        yarnConf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_AMRM_PROXY_ADDRESS);
        rmClient.init(yarnConf);
        rmClient.start();
        // Submit application
        ApplicationAttemptId appAttmptId = createApp(rmClient, cluster, conf);
        ApplicationId appId = appAttmptId.getApplicationId();
        client = createAMRMProtocol(rmClient, appId, cluster, yarnConf);
        LOG.info("testAMRMProxyE2E - Register Application Master");
        RegisterApplicationMasterResponse responseRegister = client.registerApplicationMaster(RegisterApplicationMasterRequest.newInstance(NetUtils.getHostname(), 1024, ""));
        Assert.assertNotNull(responseRegister);
        Assert.assertNotNull(responseRegister.getQueue());
        Assert.assertNotNull(responseRegister.getApplicationACLs());
        Assert.assertNotNull(responseRegister.getClientToAMTokenMasterKey());
        Assert.assertNotNull(responseRegister.getContainersFromPreviousAttempts());
        Assert.assertNotNull(responseRegister.getSchedulerResourceTypes());
        Assert.assertNotNull(responseRegister.getMaximumResourceCapability());
        RMApp rmApp = cluster.getResourceManager().getRMContext().getRMApps().get(appId);
        Assert.assertEquals(RMAppState.RUNNING, rmApp.getState());
        LOG.info("testAMRMProxyE2E - Allocate Resources Application Master");
        AllocateRequest request = createAllocateRequest(rmClient.getNodeReports(NodeState.RUNNING));
        AllocateResponse allocResponse = client.allocate(request);
        Assert.assertNotNull(allocResponse);
        Assert.assertEquals(0, allocResponse.getAllocatedContainers().size());
        request.setAskList(new ArrayList<ResourceRequest>());
        request.setResponseId(request.getResponseId() + 1);
        Thread.sleep(1000);
        // RM should allocate container within 2 calls to allocate()
        allocResponse = client.allocate(request);
        Assert.assertNotNull(allocResponse);
        Assert.assertEquals(2, allocResponse.getAllocatedContainers().size());
        LOG.info("testAMRMPRoxy - Finish Application Master");
        FinishApplicationMasterResponse responseFinish = client.finishApplicationMaster(FinishApplicationMasterRequest.newInstance(FinalApplicationStatus.SUCCEEDED, "success", null));
        Assert.assertNotNull(responseFinish);
        Thread.sleep(500);
        Assert.assertNotEquals(RMAppState.FINISHED, rmApp.getState());
    }
}

Example 23 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class TestAMRMProxy method testAMRMProxyTokenRenewal.

/*
   * This test validates the token renewal from the AMRMPRoxy. The test verifies
   * that the received token from AMRMProxy is different from the previous one
   * within 5 requests.
   */
@Test(timeout = 120000)
public void testAMRMProxyTokenRenewal() throws Exception {
    ApplicationMasterProtocol client;
    try (MiniYARNCluster cluster = new MiniYARNCluster("testE2ETokenRenewal", 1, 1, 1);
        YarnClient rmClient = YarnClient.createYarnClient()) {
        Configuration conf = new YarnConfiguration();
        conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true);
        conf.setInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 1500);
        conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 1500);
        conf.setInt(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 1500);
        // RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS should be at least
        // RM_AM_EXPIRY_INTERVAL_MS * 1.5 *3
        conf.setInt(YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, 6);
        cluster.init(conf);
        cluster.start();
        final Configuration yarnConf = cluster.getConfig();
        yarnConf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_AMRM_PROXY_ADDRESS);
        rmClient.init(yarnConf);
        rmClient.start();
        // Submit
        ApplicationAttemptId appAttmptId = createApp(rmClient, cluster, conf);
        ApplicationId appId = appAttmptId.getApplicationId();
        client = createAMRMProtocol(rmClient, appId, cluster, yarnConf);
        client.registerApplicationMaster(RegisterApplicationMasterRequest.newInstance(NetUtils.getHostname(), 1024, ""));
        LOG.info("testAMRMProxyTokenRenewal - Allocate Resources Application Master");
        AllocateRequest request = createAllocateRequest(rmClient.getNodeReports(NodeState.RUNNING));
        Token lastToken = null;
        AllocateResponse response = null;
        for (int i = 0; i < 5; i++) {
            response = client.allocate(request);
            request.setResponseId(request.getResponseId() + 1);
            if (response.getAMRMToken() != null && !response.getAMRMToken().equals(lastToken)) {
                break;
            }
            lastToken = response.getAMRMToken();
            // Time slot to be sure the AMRMProxy renew the token
            Thread.sleep(1500);
        }
        Assert.assertFalse(response.getAMRMToken().equals(lastToken));
        LOG.info("testAMRMPRoxy - Finish Application Master");
        client.finishApplicationMaster(FinishApplicationMasterRequest.newInstance(FinalApplicationStatus.SUCCEEDED, "success", null));
    }
}

Example 24 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class TestApplicationMasterServiceProtocolOnHA method testAllocateOnHA.

@Test(timeout = 15000)
public void testAllocateOnHA() throws YarnException, IOException {
    AllocateRequest request = AllocateRequest.newInstance(0, 50f, new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>(), ResourceBlacklistRequest.newInstance(new ArrayList<String>(), new ArrayList<String>()));
    AllocateResponse response = amClient.allocate(request);
    Assert.assertEquals(response, this.cluster.createFakeAllocateResponse());
}

Example 25 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class TestDistributedScheduling method testOpportunisticExecutionTypeRequestE2E.

/**
   * Validates if Allocate Requests containing only OPPORTUNISTIC container
   * requests are satisfied instantly.
   *
   * @throws Exception
   */
@Test(timeout = 60000)
public void testOpportunisticExecutionTypeRequestE2E() throws Exception {
    LOG.info("testDistributedSchedulingE2E - Register");
    RegisterApplicationMasterResponse responseRegister = client.registerApplicationMaster(RegisterApplicationMasterRequest.newInstance(NetUtils.getHostname(), 1024, ""));
    Assert.assertNotNull(responseRegister);
    Assert.assertNotNull(responseRegister.getQueue());
    Assert.assertNotNull(responseRegister.getApplicationACLs());
    Assert.assertNotNull(responseRegister.getClientToAMTokenMasterKey());
    Assert.assertNotNull(responseRegister.getContainersFromPreviousAttempts());
    Assert.assertNotNull(responseRegister.getSchedulerResourceTypes());
    Assert.assertNotNull(responseRegister.getMaximumResourceCapability());
    // Wait until the RM has been updated and verify
    Map<ApplicationId, RMApp> rmApps = cluster.getResourceManager().getRMContext().getRMApps();
    boolean rmUpdated = false;
    for (int i = 0; i < 10 && !rmUpdated; i++) {
        sleep(100);
        RMApp rmApp = rmApps.get(appId);
        if (rmApp.getState() == RMAppState.RUNNING) {
            rmUpdated = true;
        }
    }
    RMApp rmApp = rmApps.get(appId);
    Assert.assertEquals(RMAppState.RUNNING, rmApp.getState());
    LOG.info("testDistributedSchedulingE2E - Allocate");
    AllocateRequest request = createAllocateRequest(rmClient.getNodeReports(NodeState.RUNNING));
    // Replace 'ANY' requests with OPPORTUNISTIC aks and remove
    // everything else
    List<ResourceRequest> newAskList = new ArrayList<>();
    for (ResourceRequest rr : request.getAskList()) {
        if (ResourceRequest.ANY.equals(rr.getResourceName())) {
            ResourceRequest newRR = ResourceRequest.newInstance(rr.getPriority(), rr.getResourceName(), rr.getCapability(), rr.getNumContainers(), rr.getRelaxLocality(), rr.getNodeLabelExpression(), ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true));
            newAskList.add(newRR);
        }
    }
    request.setAskList(newAskList);
    AllocateResponse allocResponse = client.allocate(request);
    Assert.assertNotNull(allocResponse);
    // Ensure that all the requests are satisfied immediately
    Assert.assertEquals(2, allocResponse.getAllocatedContainers().size());
    // Verify that the allocated containers are OPPORTUNISTIC
    for (Container allocatedContainer : allocResponse.getAllocatedContainers()) {
        ContainerTokenIdentifier containerTokenIdentifier = BuilderUtils.newContainerTokenIdentifier(allocatedContainer.getContainerToken());
        Assert.assertEquals(ExecutionType.OPPORTUNISTIC, containerTokenIdentifier.getExecutionType());
    }
    // Check that the RM sees OPPORTUNISTIC containers
    ResourceScheduler scheduler = cluster.getResourceManager().getResourceScheduler();
    for (Container allocatedContainer : allocResponse.getAllocatedContainers()) {
        ContainerId containerId = allocatedContainer.getId();
        RMContainer rmContainer = scheduler.getRMContainer(containerId);
        Assert.assertEquals(ExecutionType.OPPORTUNISTIC, rmContainer.getExecutionType());
    }
    LOG.info("testDistributedSchedulingE2E - Finish");
}