Examples with NoopAMPreemptionPolicy org.apache.hadoop.mapreduce.v2.app.rm.preemption.NoopAMPreemptionPolicy used on opensource projects

Example 1 with NoopAMPreemptionPolicy

use of org.apache.hadoop.mapreduce.v2.app.rm.preemption.NoopAMPreemptionPolicy in project hadoop by apache.

the class MRAppBenchmark method benchmark1.

@Test
public void benchmark1() throws Exception {
    // Adjust for benchmarking. Start with thousands.
    int maps = 100;
    int reduces = 0;
    System.out.println("Running benchmark with maps:" + maps + " reduces:" + reduces);
    run(new MRApp(maps, reduces, true, this.getClass().getName(), true) {

        @Override
        protected ContainerAllocator createContainerAllocator(ClientService clientService, AppContext context) {
            AMPreemptionPolicy policy = new NoopAMPreemptionPolicy();
            return new RMContainerAllocator(clientService, context, policy) {

                @Override
                protected ApplicationMasterProtocol createSchedulerProxy() {
                    return new ApplicationMasterProtocol() {

                        @Override
                        public RegisterApplicationMasterResponse registerApplicationMaster(RegisterApplicationMasterRequest request) throws IOException {
                            RegisterApplicationMasterResponse response = Records.newRecord(RegisterApplicationMasterResponse.class);
                            response.setMaximumResourceCapability(Resource.newInstance(10240, 1));
                            return response;
                        }

                        @Override
                        public FinishApplicationMasterResponse finishApplicationMaster(FinishApplicationMasterRequest request) throws IOException {
                            FinishApplicationMasterResponse response = Records.newRecord(FinishApplicationMasterResponse.class);
                            return response;
                        }

                        @Override
                        public AllocateResponse allocate(AllocateRequest request) throws IOException {
                            AllocateResponse response = Records.newRecord(AllocateResponse.class);
                            List<ResourceRequest> askList = request.getAskList();
                            List<Container> containers = new ArrayList<Container>();
                            for (ResourceRequest req : askList) {
                                if (!ResourceRequest.isAnyLocation(req.getResourceName())) {
                                    continue;
                                }
                                int numContainers = req.getNumContainers();
                                for (int i = 0; i < numContainers; i++) {
                                    ContainerId containerId = ContainerId.newContainerId(getContext().getApplicationAttemptId(), request.getResponseId() + i);
                                    containers.add(Container.newInstance(containerId, NodeId.newInstance("host" + containerId.getContainerId(), 2345), "host" + containerId.getContainerId() + ":5678", req.getCapability(), req.getPriority(), null));
                                }
                            }
                            response.setAllocatedContainers(containers);
                            response.setResponseId(request.getResponseId() + 1);
                            response.setNumClusterNodes(350);
                            return response;
                        }
                    };
                }
            };
        }
    });
}

Example 2 with NoopAMPreemptionPolicy

use of org.apache.hadoop.mapreduce.v2.app.rm.preemption.NoopAMPreemptionPolicy in project hadoop by apache.

the class TestRMContainerAllocator method testHeartbeatHandler.

@Test
public void testHeartbeatHandler() throws Exception {
    LOG.info("Running testHeartbeatHandler");
    Configuration conf = new Configuration();
    conf.setInt(MRJobConfig.MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS, 1);
    ControlledClock clock = new ControlledClock();
    AppContext appContext = mock(AppContext.class);
    when(appContext.getClock()).thenReturn(clock);
    when(appContext.getApplicationID()).thenReturn(ApplicationId.newInstance(1, 1));
    RMContainerAllocator allocator = new RMContainerAllocator(mock(ClientService.class), appContext, new NoopAMPreemptionPolicy()) {

        @Override
        protected void register() {
        }

        @Override
        protected ApplicationMasterProtocol createSchedulerProxy() {
            return mock(ApplicationMasterProtocol.class);
        }

        @Override
        protected synchronized void heartbeat() throws Exception {
        }
    };
    allocator.init(conf);
    allocator.start();
    clock.setTime(5);
    int timeToWaitMs = 5000;
    while (allocator.getLastHeartbeatTime() != 5 && timeToWaitMs > 0) {
        Thread.sleep(10);
        timeToWaitMs -= 10;
    }
    Assert.assertEquals(5, allocator.getLastHeartbeatTime());
    clock.setTime(7);
    timeToWaitMs = 5000;
    while (allocator.getLastHeartbeatTime() != 7 && timeToWaitMs > 0) {
        Thread.sleep(10);
        timeToWaitMs -= 10;
    }
    Assert.assertEquals(7, allocator.getLastHeartbeatTime());
    final AtomicBoolean callbackCalled = new AtomicBoolean(false);
    allocator.runOnNextHeartbeat(new Runnable() {

        @Override
        public void run() {
            callbackCalled.set(true);
        }
    });
    clock.setTime(8);
    timeToWaitMs = 5000;
    while (allocator.getLastHeartbeatTime() != 8 && timeToWaitMs > 0) {
        Thread.sleep(10);
        timeToWaitMs -= 10;
    }
    Assert.assertEquals(8, allocator.getLastHeartbeatTime());
    Assert.assertTrue(callbackCalled.get());
}

Example 3 with NoopAMPreemptionPolicy

use of org.apache.hadoop.mapreduce.v2.app.rm.preemption.NoopAMPreemptionPolicy in project hadoop by apache.

the class TestRMContainerAllocator method testCompletedContainerEvent.

@Test
public void testCompletedContainerEvent() {
    RMContainerAllocator allocator = new RMContainerAllocator(mock(ClientService.class), mock(AppContext.class), new NoopAMPreemptionPolicy());
    TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(MRBuilderUtils.newTaskId(MRBuilderUtils.newJobId(1, 1, 1), 1, TaskType.MAP), 1);
    ApplicationId applicationId = ApplicationId.newInstance(1, 1);
    ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
    ContainerId containerId = ContainerId.newContainerId(applicationAttemptId, 1);
    ContainerStatus status = ContainerStatus.newInstance(containerId, ContainerState.RUNNING, "", 0);
    ContainerStatus abortedStatus = ContainerStatus.newInstance(containerId, ContainerState.RUNNING, "", ContainerExitStatus.ABORTED);
    TaskAttemptEvent event = allocator.createContainerFinishedEvent(status, attemptId);
    Assert.assertEquals(TaskAttemptEventType.TA_CONTAINER_COMPLETED, event.getType());
    TaskAttemptEvent abortedEvent = allocator.createContainerFinishedEvent(abortedStatus, attemptId);
    Assert.assertEquals(TaskAttemptEventType.TA_KILL, abortedEvent.getType());
    ContainerId containerId2 = ContainerId.newContainerId(applicationAttemptId, 2);
    ContainerStatus status2 = ContainerStatus.newInstance(containerId2, ContainerState.RUNNING, "", 0);
    ContainerStatus preemptedStatus = ContainerStatus.newInstance(containerId2, ContainerState.RUNNING, "", ContainerExitStatus.PREEMPTED);
    TaskAttemptEvent event2 = allocator.createContainerFinishedEvent(status2, attemptId);
    Assert.assertEquals(TaskAttemptEventType.TA_CONTAINER_COMPLETED, event2.getType());
    TaskAttemptEvent abortedEvent2 = allocator.createContainerFinishedEvent(preemptedStatus, attemptId);
    Assert.assertEquals(TaskAttemptEventType.TA_KILL, abortedEvent2.getType());
}