Examples with NodeResourceUpdateSchedulerEvent org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeResourceUpdateSchedulerEvent used on opensource projects

Example 1 with NodeResourceUpdateSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeResourceUpdateSchedulerEvent in project hadoop by apache.

the class FairScheduler method handle.

@Override
public void handle(SchedulerEvent event) {
    switch(event.getType()) {
        case NODE_ADDED:
            if (!(event instanceof NodeAddedSchedulerEvent)) {
                throw new RuntimeException("Unexpected event type: " + event);
            }
            NodeAddedSchedulerEvent nodeAddedEvent = (NodeAddedSchedulerEvent) event;
            addNode(nodeAddedEvent.getContainerReports(), nodeAddedEvent.getAddedRMNode());
            break;
        case NODE_REMOVED:
            if (!(event instanceof NodeRemovedSchedulerEvent)) {
                throw new RuntimeException("Unexpected event type: " + event);
            }
            NodeRemovedSchedulerEvent nodeRemovedEvent = (NodeRemovedSchedulerEvent) event;
            removeNode(nodeRemovedEvent.getRemovedRMNode());
            break;
        case NODE_UPDATE:
            if (!(event instanceof NodeUpdateSchedulerEvent)) {
                throw new RuntimeException("Unexpected event type: " + event);
            }
            NodeUpdateSchedulerEvent nodeUpdatedEvent = (NodeUpdateSchedulerEvent) event;
            nodeUpdate(nodeUpdatedEvent.getRMNode());
            break;
        case APP_ADDED:
            if (!(event instanceof AppAddedSchedulerEvent)) {
                throw new RuntimeException("Unexpected event type: " + event);
            }
            AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent) event;
            String queueName = resolveReservationQueueName(appAddedEvent.getQueue(), appAddedEvent.getApplicationId(), appAddedEvent.getReservationID(), appAddedEvent.getIsAppRecovering());
            if (queueName != null) {
                addApplication(appAddedEvent.getApplicationId(), queueName, appAddedEvent.getUser(), appAddedEvent.getIsAppRecovering());
            }
            break;
        case APP_REMOVED:
            if (!(event instanceof AppRemovedSchedulerEvent)) {
                throw new RuntimeException("Unexpected event type: " + event);
            }
            AppRemovedSchedulerEvent appRemovedEvent = (AppRemovedSchedulerEvent) event;
            removeApplication(appRemovedEvent.getApplicationID(), appRemovedEvent.getFinalState());
            break;
        case NODE_RESOURCE_UPDATE:
            if (!(event instanceof NodeResourceUpdateSchedulerEvent)) {
                throw new RuntimeException("Unexpected event type: " + event);
            }
            NodeResourceUpdateSchedulerEvent nodeResourceUpdatedEvent = (NodeResourceUpdateSchedulerEvent) event;
            updateNodeResource(nodeResourceUpdatedEvent.getRMNode(), nodeResourceUpdatedEvent.getResourceOption());
            break;
        case APP_ATTEMPT_ADDED:
            if (!(event instanceof AppAttemptAddedSchedulerEvent)) {
                throw new RuntimeException("Unexpected event type: " + event);
            }
            AppAttemptAddedSchedulerEvent appAttemptAddedEvent = (AppAttemptAddedSchedulerEvent) event;
            addApplicationAttempt(appAttemptAddedEvent.getApplicationAttemptId(), appAttemptAddedEvent.getTransferStateFromPreviousAttempt(), appAttemptAddedEvent.getIsAttemptRecovering());
            break;
        case APP_ATTEMPT_REMOVED:
            if (!(event instanceof AppAttemptRemovedSchedulerEvent)) {
                throw new RuntimeException("Unexpected event type: " + event);
            }
            AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent = (AppAttemptRemovedSchedulerEvent) event;
            removeApplicationAttempt(appAttemptRemovedEvent.getApplicationAttemptID(), appAttemptRemovedEvent.getFinalAttemptState(), appAttemptRemovedEvent.getKeepContainersAcrossAppAttempts());
            break;
        case CONTAINER_EXPIRED:
            if (!(event instanceof ContainerExpiredSchedulerEvent)) {
                throw new RuntimeException("Unexpected event type: " + event);
            }
            ContainerExpiredSchedulerEvent containerExpiredEvent = (ContainerExpiredSchedulerEvent) event;
            ContainerId containerId = containerExpiredEvent.getContainerId();
            super.completedContainer(getRMContainer(containerId), SchedulerUtils.createAbnormalContainerStatus(containerId, SchedulerUtils.EXPIRED_CONTAINER), RMContainerEventType.EXPIRE);
            break;
        default:
            LOG.error("Unknown event arrived at FairScheduler: " + event.toString());
    }
}

Example 2 with NodeResourceUpdateSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeResourceUpdateSchedulerEvent in project hadoop by apache.

the class TestFifoScheduler method testUpdateResourceOnNode.

@Test(timeout = 2000)
public void testUpdateResourceOnNode() throws Exception {
    AsyncDispatcher dispatcher = new InlineDispatcher();
    Configuration conf = new Configuration();
    RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(conf);
    containerTokenSecretManager.rollMasterKey();
    NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf);
    nmTokenSecretManager.rollMasterKey();
    RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
    FifoScheduler scheduler = new FifoScheduler();
    RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null, containerTokenSecretManager, nmTokenSecretManager, null, scheduler);
    rmContext.setSystemMetricsPublisher(mock(SystemMetricsPublisher.class));
    rmContext.setRMApplicationHistoryWriter(mock(RMApplicationHistoryWriter.class));
    ((RMContextImpl) rmContext).setYarnConfiguration(new YarnConfiguration());
    NullRMNodeLabelsManager nlm = new NullRMNodeLabelsManager();
    nlm.init(new Configuration());
    rmContext.setNodeLabelManager(nlm);
    scheduler.setRMContext(rmContext);
    ((RMContextImpl) rmContext).setScheduler(scheduler);
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(new Configuration(), rmContext);
    RMNode node0 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 4), 1, "127.0.0.1");
    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
    scheduler.handle(nodeEvent1);
    assertEquals(scheduler.getNumClusterNodes(), 1);
    Resource newResource = Resources.createResource(1024, 4);
    NodeResourceUpdateSchedulerEvent node0ResourceUpdate = new NodeResourceUpdateSchedulerEvent(node0, ResourceOption.newInstance(newResource, ResourceOption.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT));
    scheduler.handle(node0ResourceUpdate);
    // SchedulerNode's total resource and available resource are changed.
    assertEquals(1024, scheduler.getNodeTracker().getNode(node0.getNodeID()).getTotalResource().getMemorySize());
    assertEquals(1024, scheduler.getNodeTracker().getNode(node0.getNodeID()).getUnallocatedResource().getMemorySize(), 1024);
    QueueInfo queueInfo = scheduler.getQueueInfo(null, false, false);
    Assert.assertEquals(0.0f, queueInfo.getCurrentCapacity(), 0.0f);
    int _appId = 1;
    int _appAttemptId = 1;
    ApplicationAttemptId appAttemptId = createAppAttemptId(_appId, _appAttemptId);
    createMockRMApp(appAttemptId, rmContext);
    AppAddedSchedulerEvent appEvent = new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "queue1", "user1");
    scheduler.handle(appEvent);
    AppAttemptAddedSchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false);
    scheduler.handle(attemptEvent);
    int memory = 1024;
    int priority = 1;
    List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
    ResourceRequest nodeLocal = createResourceRequest(memory, node0.getHostName(), priority, 1);
    ResourceRequest rackLocal = createResourceRequest(memory, node0.getRackName(), priority, 1);
    ResourceRequest any = createResourceRequest(memory, ResourceRequest.ANY, priority, 1);
    ask.add(nodeLocal);
    ask.add(rackLocal);
    ask.add(any);
    scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS);
    // Before the node update event, there are one local request
    Assert.assertEquals(1, nodeLocal.getNumContainers());
    NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
    // Now schedule.
    scheduler.handle(node0Update);
    // After the node update event, check no local request
    Assert.assertEquals(0, nodeLocal.getNumContainers());
    // Also check that one container was scheduled
    SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId);
    Assert.assertEquals(1, info.getLiveContainers().size());
    // And check the default Queue now is full.
    queueInfo = scheduler.getQueueInfo(null, false, false);
    Assert.assertEquals(1.0f, queueInfo.getCurrentCapacity(), 0.0f);
}

Example 3 with NodeResourceUpdateSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeResourceUpdateSchedulerEvent in project hadoop by apache.

the class OpportunisticContainerAllocatorAMService method handle.

@Override
public void handle(SchedulerEvent event) {
    switch(event.getType()) {
        case NODE_ADDED:
            if (!(event instanceof NodeAddedSchedulerEvent)) {
                throw new RuntimeException("Unexpected event type: " + event);
            }
            NodeAddedSchedulerEvent nodeAddedEvent = (NodeAddedSchedulerEvent) event;
            nodeMonitor.addNode(nodeAddedEvent.getContainerReports(), nodeAddedEvent.getAddedRMNode());
            break;
        case NODE_REMOVED:
            if (!(event instanceof NodeRemovedSchedulerEvent)) {
                throw new RuntimeException("Unexpected event type: " + event);
            }
            NodeRemovedSchedulerEvent nodeRemovedEvent = (NodeRemovedSchedulerEvent) event;
            nodeMonitor.removeNode(nodeRemovedEvent.getRemovedRMNode());
            break;
        case NODE_UPDATE:
            if (!(event instanceof NodeUpdateSchedulerEvent)) {
                throw new RuntimeException("Unexpected event type: " + event);
            }
            NodeUpdateSchedulerEvent nodeUpdatedEvent = (NodeUpdateSchedulerEvent) event;
            nodeMonitor.updateNode(nodeUpdatedEvent.getRMNode());
            break;
        case NODE_RESOURCE_UPDATE:
            if (!(event instanceof NodeResourceUpdateSchedulerEvent)) {
                throw new RuntimeException("Unexpected event type: " + event);
            }
            NodeResourceUpdateSchedulerEvent nodeResourceUpdatedEvent = (NodeResourceUpdateSchedulerEvent) event;
            nodeMonitor.updateNodeResource(nodeResourceUpdatedEvent.getRMNode(), nodeResourceUpdatedEvent.getResourceOption());
            break;
        // <-- IGNORED EVENTS : START -->
        case APP_ADDED:
            break;
        case APP_REMOVED:
            break;
        case APP_ATTEMPT_ADDED:
            break;
        case APP_ATTEMPT_REMOVED:
            break;
        case CONTAINER_EXPIRED:
            break;
        case NODE_LABELS_UPDATE:
            break;
        // <-- IGNORED EVENTS : END -->
        default:
            LOG.error("Unknown event arrived at" + "OpportunisticContainerAllocatorAMService: " + event.toString());
    }
}

Example 4 with NodeResourceUpdateSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeResourceUpdateSchedulerEvent in project incubator-myriad by apache.

the class YarnNodeCapacityManager method setNodeCapacity.

/**
 * 1. Updates {@link RMNode#getTotalCapability()} with newCapacity.
 * 2. Sends out a {@link NodeResourceUpdateSchedulerEvent} that's handled by YARN's scheduler.
 * The scheduler updates the corresponding {@link SchedulerNode} with the newCapacity.
 *
 * @param rmNode
 * @param newCapacity
 */
@SuppressWarnings("unchecked")
public void setNodeCapacity(RMNode rmNode, Resource newCapacity) {
    // NOOP prevent YARN warning changing to same size
    if ((Resources.equals(rmNode.getTotalCapability(), newCapacity))) {
        return;
    }
    if (yarnScheduler.getSchedulerNode(rmNode.getNodeID()) == null) {
        LOGGER.info("Yarn Scheduler doesn't have node {}, probably UNHEALTHY", rmNode.getNodeID());
        return;
    }
    yarnSchedulerLock.lock();
    try {
        if (newCapacity.getMemory() < 0 || newCapacity.getVirtualCores() < 0) {
            Resource zeroed = ResourceUtils.componentwiseMax(ZERO_RESOURCE, newCapacity);
            rmNode.getTotalCapability().setMemory(zeroed.getMemory());
            rmNode.getTotalCapability().setVirtualCores(zeroed.getVirtualCores());
            LOGGER.warn("Asked to set Node {} to a value less than zero!  Had {}, setting to {}.", rmNode.getHttpAddress(), rmNode.getTotalCapability().toString(), zeroed.toString());
        } else {
            rmNode.getTotalCapability().setMemory(newCapacity.getMemory());
            rmNode.getTotalCapability().setVirtualCores(newCapacity.getVirtualCores());
            if (LOGGER.isInfoEnabled()) {
                LOGGER.info("Setting capacity for node {} to {}", rmNode.getHostName(), newCapacity);
            }
        }
        // updates the scheduler with the new capacity for the NM.
        // the event is handled by the scheduler asynchronously
        rmContext.getDispatcher().getEventHandler().handle(new NodeResourceUpdateSchedulerEvent(rmNode, ResourceOption.newInstance(rmNode.getTotalCapability(), RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
    } finally {
        yarnSchedulerLock.unlock();
    }
}