Examples with NodeRemovedSchedulerEvent org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent used on opensource projects

Example 11 with NodeRemovedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent in project hadoop by apache.

the class RMNodeImpl method reportNodeUnusable.

/**
   * Report node is UNUSABLE and update metrics.
   * @param rmNode
   * @param finalState
   */
public static void reportNodeUnusable(RMNodeImpl rmNode, NodeState finalState) {
    // Inform the scheduler
    rmNode.nodeUpdateQueue.clear();
    // If the current state is NodeState.UNHEALTHY
    // Then node is already been removed from the
    // Scheduler
    NodeState initialState = rmNode.getState();
    if (!initialState.equals(NodeState.UNHEALTHY)) {
        rmNode.context.getDispatcher().getEventHandler().handle(new NodeRemovedSchedulerEvent(rmNode));
    }
    rmNode.context.getDispatcher().getEventHandler().handle(new NodesListManagerEvent(NodesListManagerEventType.NODE_UNUSABLE, rmNode));
    //Update the metrics
    rmNode.updateMetricsForDeactivatedNode(initialState, finalState);
}

Example 12 with NodeRemovedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent in project hadoop by apache.

the class TestAbstractYarnScheduler method testMaximumAllocationVCoresHelper.

private void testMaximumAllocationVCoresHelper(YarnScheduler scheduler, final int node1MaxVCores, final int node2MaxVCores, final int node3MaxVCores, final int... expectedMaxVCores) throws Exception {
    Assert.assertEquals(6, expectedMaxVCores.length);
    Assert.assertEquals(0, scheduler.getNumClusterNodes());
    int maxVCores = scheduler.getMaximumResourceCapability().getVirtualCores();
    Assert.assertEquals(expectedMaxVCores[0], maxVCores);
    RMNode node1 = MockNodes.newNodeInfo(0, Resources.createResource(1024, node1MaxVCores), 1, "127.0.0.2");
    scheduler.handle(new NodeAddedSchedulerEvent(node1));
    Assert.assertEquals(1, scheduler.getNumClusterNodes());
    maxVCores = scheduler.getMaximumResourceCapability().getVirtualCores();
    Assert.assertEquals(expectedMaxVCores[1], maxVCores);
    scheduler.handle(new NodeRemovedSchedulerEvent(node1));
    Assert.assertEquals(0, scheduler.getNumClusterNodes());
    maxVCores = scheduler.getMaximumResourceCapability().getVirtualCores();
    Assert.assertEquals(expectedMaxVCores[2], maxVCores);
    RMNode node2 = MockNodes.newNodeInfo(0, Resources.createResource(1024, node2MaxVCores), 2, "127.0.0.3");
    scheduler.handle(new NodeAddedSchedulerEvent(node2));
    Assert.assertEquals(1, scheduler.getNumClusterNodes());
    maxVCores = scheduler.getMaximumResourceCapability().getVirtualCores();
    Assert.assertEquals(expectedMaxVCores[3], maxVCores);
    RMNode node3 = MockNodes.newNodeInfo(0, Resources.createResource(1024, node3MaxVCores), 3, "127.0.0.4");
    scheduler.handle(new NodeAddedSchedulerEvent(node3));
    Assert.assertEquals(2, scheduler.getNumClusterNodes());
    maxVCores = scheduler.getMaximumResourceCapability().getVirtualCores();
    Assert.assertEquals(expectedMaxVCores[4], maxVCores);
    scheduler.handle(new NodeRemovedSchedulerEvent(node3));
    Assert.assertEquals(1, scheduler.getNumClusterNodes());
    maxVCores = scheduler.getMaximumResourceCapability().getVirtualCores();
    Assert.assertEquals(expectedMaxVCores[5], maxVCores);
    scheduler.handle(new NodeRemovedSchedulerEvent(node2));
    Assert.assertEquals(0, scheduler.getNumClusterNodes());
}

Example 13 with NodeRemovedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent in project hadoop by apache.

the class TestCapacityScheduler method testSchedulingOnRemovedNode.

@Test
public void testSchedulingOnRemovedNode() throws Exception {
    Configuration conf = new YarnConfiguration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    conf.setBoolean(CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, false);
    MockRM rm = new MockRM(conf);
    rm.start();
    RMApp app = rm.submitApp(100);
    rm.drainEvents();
    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 10240, 10);
    MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1);
    //remove nm2 to keep am alive
    MockNM nm2 = rm.registerNode("127.0.0.1:1235", 10240, 10);
    am.allocate(ResourceRequest.ANY, 2048, 1, null);
    CapacityScheduler scheduler = (CapacityScheduler) rm.getRMContext().getScheduler();
    FiCaSchedulerNode node = (FiCaSchedulerNode) scheduler.getNodeTracker().getNode(nm2.getNodeId());
    scheduler.handle(new NodeRemovedSchedulerEvent(rm.getRMContext().getRMNodes().get(nm2.getNodeId())));
    // schedulerNode is removed, try allocate a container
    scheduler.allocateContainersToNode(new SimplePlacementSet<>(node), true);
    AppAttemptRemovedSchedulerEvent appRemovedEvent1 = new AppAttemptRemovedSchedulerEvent(am.getApplicationAttemptId(), RMAppAttemptState.FINISHED, false);
    scheduler.handle(appRemovedEvent1);
    rm.stop();
}

Example 14 with NodeRemovedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent in project hadoop by apache.

the class TestCapacityScheduler method testReconnectedNode.

@Test
public void testReconnectedNode() throws Exception {
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    setupQueueConfiguration(csConf);
    CapacityScheduler cs = new CapacityScheduler();
    cs.setConf(new YarnConfiguration());
    cs.setRMContext(resourceManager.getRMContext());
    cs.init(csConf);
    cs.start();
    cs.reinitialize(csConf, new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null));
    RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1);
    RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2);
    cs.handle(new NodeAddedSchedulerEvent(n1));
    cs.handle(new NodeAddedSchedulerEvent(n2));
    Assert.assertEquals(6 * GB, cs.getClusterResource().getMemorySize());
    // reconnect n1 with downgraded memory
    n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 1);
    cs.handle(new NodeRemovedSchedulerEvent(n1));
    cs.handle(new NodeAddedSchedulerEvent(n1));
    Assert.assertEquals(4 * GB, cs.getClusterResource().getMemorySize());
    cs.stop();
}

Example 15 with NodeRemovedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent in project hadoop by apache.

the class FairScheduler method handle.

@Override
public void handle(SchedulerEvent event) {
    switch(event.getType()) {
        case NODE_ADDED:
            if (!(event instanceof NodeAddedSchedulerEvent)) {
                throw new RuntimeException("Unexpected event type: " + event);
            }
            NodeAddedSchedulerEvent nodeAddedEvent = (NodeAddedSchedulerEvent) event;
            addNode(nodeAddedEvent.getContainerReports(), nodeAddedEvent.getAddedRMNode());
            break;
        case NODE_REMOVED:
            if (!(event instanceof NodeRemovedSchedulerEvent)) {
                throw new RuntimeException("Unexpected event type: " + event);
            }
            NodeRemovedSchedulerEvent nodeRemovedEvent = (NodeRemovedSchedulerEvent) event;
            removeNode(nodeRemovedEvent.getRemovedRMNode());
            break;
        case NODE_UPDATE:
            if (!(event instanceof NodeUpdateSchedulerEvent)) {
                throw new RuntimeException("Unexpected event type: " + event);
            }
            NodeUpdateSchedulerEvent nodeUpdatedEvent = (NodeUpdateSchedulerEvent) event;
            nodeUpdate(nodeUpdatedEvent.getRMNode());
            break;
        case APP_ADDED:
            if (!(event instanceof AppAddedSchedulerEvent)) {
                throw new RuntimeException("Unexpected event type: " + event);
            }
            AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent) event;
            String queueName = resolveReservationQueueName(appAddedEvent.getQueue(), appAddedEvent.getApplicationId(), appAddedEvent.getReservationID(), appAddedEvent.getIsAppRecovering());
            if (queueName != null) {
                addApplication(appAddedEvent.getApplicationId(), queueName, appAddedEvent.getUser(), appAddedEvent.getIsAppRecovering());
            }
            break;
        case APP_REMOVED:
            if (!(event instanceof AppRemovedSchedulerEvent)) {
                throw new RuntimeException("Unexpected event type: " + event);
            }
            AppRemovedSchedulerEvent appRemovedEvent = (AppRemovedSchedulerEvent) event;
            removeApplication(appRemovedEvent.getApplicationID(), appRemovedEvent.getFinalState());
            break;
        case NODE_RESOURCE_UPDATE:
            if (!(event instanceof NodeResourceUpdateSchedulerEvent)) {
                throw new RuntimeException("Unexpected event type: " + event);
            }
            NodeResourceUpdateSchedulerEvent nodeResourceUpdatedEvent = (NodeResourceUpdateSchedulerEvent) event;
            updateNodeResource(nodeResourceUpdatedEvent.getRMNode(), nodeResourceUpdatedEvent.getResourceOption());
            break;
        case APP_ATTEMPT_ADDED:
            if (!(event instanceof AppAttemptAddedSchedulerEvent)) {
                throw new RuntimeException("Unexpected event type: " + event);
            }
            AppAttemptAddedSchedulerEvent appAttemptAddedEvent = (AppAttemptAddedSchedulerEvent) event;
            addApplicationAttempt(appAttemptAddedEvent.getApplicationAttemptId(), appAttemptAddedEvent.getTransferStateFromPreviousAttempt(), appAttemptAddedEvent.getIsAttemptRecovering());
            break;
        case APP_ATTEMPT_REMOVED:
            if (!(event instanceof AppAttemptRemovedSchedulerEvent)) {
                throw new RuntimeException("Unexpected event type: " + event);
            }
            AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent = (AppAttemptRemovedSchedulerEvent) event;
            removeApplicationAttempt(appAttemptRemovedEvent.getApplicationAttemptID(), appAttemptRemovedEvent.getFinalAttemptState(), appAttemptRemovedEvent.getKeepContainersAcrossAppAttempts());
            break;
        case CONTAINER_EXPIRED:
            if (!(event instanceof ContainerExpiredSchedulerEvent)) {
                throw new RuntimeException("Unexpected event type: " + event);
            }
            ContainerExpiredSchedulerEvent containerExpiredEvent = (ContainerExpiredSchedulerEvent) event;
            ContainerId containerId = containerExpiredEvent.getContainerId();
            super.completedContainer(getRMContainer(containerId), SchedulerUtils.createAbnormalContainerStatus(containerId, SchedulerUtils.EXPIRED_CONTAINER), RMContainerEventType.EXPIRE);
            break;
        default:
            LOG.error("Unknown event arrived at FairScheduler: " + event.toString());
    }
}