Examples with Resource org.apache.hadoop.yarn.api.records.Resource used on opensource projects

Example 26 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class TestResourceCalculator method testResourceCalculatorCompareMethod.

@Test(timeout = 10000)
public void testResourceCalculatorCompareMethod() {
    Resource clusterResource = Resource.newInstance(0, 0);
    // For lhs == rhs
    Resource lhs = Resource.newInstance(0, 0);
    Resource rhs = Resource.newInstance(0, 0);
    assertResourcesOperations(clusterResource, lhs, rhs, false, true, false, true, lhs, lhs);
    // lhs > rhs
    lhs = Resource.newInstance(1, 1);
    rhs = Resource.newInstance(0, 0);
    assertResourcesOperations(clusterResource, lhs, rhs, false, false, true, true, lhs, rhs);
    // For lhs < rhs
    lhs = Resource.newInstance(0, 0);
    rhs = Resource.newInstance(1, 1);
    assertResourcesOperations(clusterResource, lhs, rhs, true, true, false, false, rhs, lhs);
    if (!(resourceCalculator instanceof DominantResourceCalculator)) {
        return;
    }
    // verify for 2 dimensional resources i.e memory and cpu
    // dominant resource types
    lhs = Resource.newInstance(1, 0);
    rhs = Resource.newInstance(0, 1);
    assertResourcesOperations(clusterResource, lhs, rhs, false, true, false, true, lhs, lhs);
    lhs = Resource.newInstance(0, 1);
    rhs = Resource.newInstance(1, 0);
    assertResourcesOperations(clusterResource, lhs, rhs, false, true, false, true, lhs, lhs);
    lhs = Resource.newInstance(1, 1);
    rhs = Resource.newInstance(1, 0);
    assertResourcesOperations(clusterResource, lhs, rhs, false, false, true, true, lhs, rhs);
    lhs = Resource.newInstance(0, 1);
    rhs = Resource.newInstance(1, 1);
    assertResourcesOperations(clusterResource, lhs, rhs, true, true, false, false, rhs, lhs);
}

Example 27 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class TestResourceCalculator method testNormalize.

/**
   * Test resource normalization.
   */
@Test(timeout = 10000)
public void testNormalize() {
    // requested resources value cannot be an arbitrary number.
    Resource ask = Resource.newInstance(1111, 2);
    Resource min = Resource.newInstance(1024, 1);
    Resource max = Resource.newInstance(8 * 1024, 8);
    Resource increment = Resource.newInstance(1024, 4);
    if (resourceCalculator instanceof DefaultResourceCalculator) {
        Resource result = Resources.normalize(resourceCalculator, ask, min, max, increment);
        Assert.assertEquals(2 * 1024, result.getMemorySize());
    } else if (resourceCalculator instanceof DominantResourceCalculator) {
        Resource result = Resources.normalize(resourceCalculator, ask, min, max, increment);
        Assert.assertEquals(2 * 1024, result.getMemorySize());
        Assert.assertEquals(4, result.getVirtualCores());
    }
    // if resources asked are less than minimum resource, then normalize it to
    // minimum resource.
    ask = Resource.newInstance(512, 0);
    min = Resource.newInstance(2 * 1024, 2);
    max = Resource.newInstance(8 * 1024, 8);
    increment = Resource.newInstance(1024, 1);
    if (resourceCalculator instanceof DefaultResourceCalculator) {
        Resource result = Resources.normalize(resourceCalculator, ask, min, max, increment);
        Assert.assertEquals(2 * 1024, result.getMemorySize());
    } else if (resourceCalculator instanceof DominantResourceCalculator) {
        Resource result = Resources.normalize(resourceCalculator, ask, min, max, increment);
        Assert.assertEquals(2 * 1024, result.getMemorySize());
        Assert.assertEquals(2, result.getVirtualCores());
    }
    // if resources asked are larger than maximum resource, then normalize it to
    // maximum resources.
    ask = Resource.newInstance(9 * 1024, 9);
    min = Resource.newInstance(2 * 1024, 2);
    max = Resource.newInstance(8 * 1024, 8);
    increment = Resource.newInstance(1024, 1);
    if (resourceCalculator instanceof DefaultResourceCalculator) {
        Resource result = Resources.normalize(resourceCalculator, ask, min, max, increment);
        Assert.assertEquals(8 * 1024, result.getMemorySize());
    } else if (resourceCalculator instanceof DominantResourceCalculator) {
        Resource result = Resources.normalize(resourceCalculator, ask, min, max, increment);
        Assert.assertEquals(8 * 1024, result.getMemorySize());
        Assert.assertEquals(8, result.getVirtualCores());
    }
    // if increment is 0, use minimum resource as the increment resource.
    ask = Resource.newInstance(1111, 2);
    min = Resource.newInstance(2 * 1024, 2);
    max = Resource.newInstance(8 * 1024, 8);
    increment = Resource.newInstance(0, 0);
    if (resourceCalculator instanceof DefaultResourceCalculator) {
        Resource result = Resources.normalize(resourceCalculator, ask, min, max, increment);
        Assert.assertEquals(2 * 1024, result.getMemorySize());
    } else if (resourceCalculator instanceof DominantResourceCalculator) {
        Resource result = Resources.normalize(resourceCalculator, ask, min, max, increment);
        Assert.assertEquals(2 * 1024, result.getMemorySize());
        Assert.assertEquals(2, result.getVirtualCores());
    }
}

Example 28 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class ContainerImpl method setResource.

@Override
public void setResource(Resource targetResource) {
    Resource currentResource = getResource();
    this.resource = Resources.clone(targetResource);
    this.metrics.changeContainer(currentResource, targetResource);
}

Example 29 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class RMServerUtils method checkSchedContainerChangeRequest.

/**
   * Validate increase/decrease request.
   * <pre>
   * - Throw exception when any other error happens
   * </pre>
   */
public static void checkSchedContainerChangeRequest(SchedContainerChangeRequest request, boolean increase) throws InvalidResourceRequestException {
    RMContext rmContext = request.getRmContext();
    ContainerId containerId = request.getContainerId();
    RMContainer rmContainer = request.getRMContainer();
    Resource targetResource = request.getTargetCapacity();
    // Compare targetResource and original resource
    Resource originalResource = rmContainer.getAllocatedResource();
    // <20G, 8>
    if (increase) {
        if (originalResource.getMemorySize() > targetResource.getMemorySize() || originalResource.getVirtualCores() > targetResource.getVirtualCores()) {
            String msg = "Trying to increase a container, but target resource has some" + " resource < original resource, target=" + targetResource + " original=" + originalResource + " containerId=" + containerId;
            throw new InvalidResourceRequestException(msg);
        }
    } else {
        if (originalResource.getMemorySize() < targetResource.getMemorySize() || originalResource.getVirtualCores() < targetResource.getVirtualCores()) {
            String msg = "Trying to decrease a container, but target resource has " + "some resource > original resource, target=" + targetResource + " original=" + originalResource + " containerId=" + containerId;
            throw new InvalidResourceRequestException(msg);
        }
    }
    // Target resource of the increase request is more than NM can offer
    ResourceScheduler scheduler = rmContext.getScheduler();
    RMNode rmNode = request.getSchedulerNode().getRMNode();
    if (!Resources.fitsIn(scheduler.getResourceCalculator(), scheduler.getClusterResource(), targetResource, rmNode.getTotalCapability())) {
        String msg = "Target resource=" + targetResource + " of containerId=" + containerId + " is more than node's total resource=" + rmNode.getTotalCapability();
        throw new InvalidResourceRequestException(msg);
    }
}

Example 30 with Resource

use of org.apache.hadoop.yarn.api.records.Resource in project hadoop by apache.

the class ResourceTrackerService method registerNodeManager.

@SuppressWarnings("unchecked")
@Override
public RegisterNodeManagerResponse registerNodeManager(RegisterNodeManagerRequest request) throws YarnException, IOException {
    NodeId nodeId = request.getNodeId();
    String host = nodeId.getHost();
    int cmPort = nodeId.getPort();
    int httpPort = request.getHttpPort();
    Resource capability = request.getResource();
    String nodeManagerVersion = request.getNMVersion();
    Resource physicalResource = request.getPhysicalResource();
    RegisterNodeManagerResponse response = recordFactory.newRecordInstance(RegisterNodeManagerResponse.class);
    if (!minimumNodeManagerVersion.equals("NONE")) {
        if (minimumNodeManagerVersion.equals("EqualToRM")) {
            minimumNodeManagerVersion = YarnVersionInfo.getVersion();
        }
        if ((nodeManagerVersion == null) || (VersionUtil.compareVersions(nodeManagerVersion, minimumNodeManagerVersion)) < 0) {
            String message = "Disallowed NodeManager Version " + nodeManagerVersion + ", is less than the minimum version " + minimumNodeManagerVersion + " sending SHUTDOWN signal to " + "NodeManager.";
            LOG.info(message);
            response.setDiagnosticsMessage(message);
            response.setNodeAction(NodeAction.SHUTDOWN);
            return response;
        }
    }
    // Check if this node is a 'valid' node
    if (!this.nodesListManager.isValidNode(host) && !isNodeInDecommissioning(nodeId)) {
        String message = "Disallowed NodeManager from  " + host + ", Sending SHUTDOWN signal to the NodeManager.";
        LOG.info(message);
        response.setDiagnosticsMessage(message);
        response.setNodeAction(NodeAction.SHUTDOWN);
        return response;
    }
    // check if node's capacity is load from dynamic-resources.xml
    String nid = nodeId.toString();
    Resource dynamicLoadCapability = loadNodeResourceFromDRConfiguration(nid);
    if (dynamicLoadCapability != null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Resource for node: " + nid + " is adjusted from: " + capability + " to: " + dynamicLoadCapability + " due to settings in dynamic-resources.xml.");
        }
        capability = dynamicLoadCapability;
        // sync back with new resource.
        response.setResource(capability);
    }
    // Check if this node has minimum allocations
    if (capability.getMemorySize() < minAllocMb || capability.getVirtualCores() < minAllocVcores) {
        String message = "NodeManager from  " + host + " doesn't satisfy minimum allocations, Sending SHUTDOWN" + " signal to the NodeManager.";
        LOG.info(message);
        response.setDiagnosticsMessage(message);
        response.setNodeAction(NodeAction.SHUTDOWN);
        return response;
    }
    response.setContainerTokenMasterKey(containerTokenSecretManager.getCurrentKey());
    response.setNMTokenMasterKey(nmTokenSecretManager.getCurrentKey());
    RMNode rmNode = new RMNodeImpl(nodeId, rmContext, host, cmPort, httpPort, resolve(host), capability, nodeManagerVersion, physicalResource);
    RMNode oldNode = this.rmContext.getRMNodes().putIfAbsent(nodeId, rmNode);
    if (oldNode == null) {
        this.rmContext.getDispatcher().getEventHandler().handle(new RMNodeStartedEvent(nodeId, request.getNMContainerStatuses(), request.getRunningApplications()));
    } else {
        LOG.info("Reconnect from the node at: " + host);
        this.nmLivelinessMonitor.unregister(nodeId);
        // Reset heartbeat ID since node just restarted.
        oldNode.resetLastNodeHeartBeatResponse();
        this.rmContext.getDispatcher().getEventHandler().handle(new RMNodeReconnectEvent(nodeId, rmNode, request.getRunningApplications(), request.getNMContainerStatuses()));
    }
    // On every node manager register we will be clearing NMToken keys if
    // present for any running application.
    this.nmTokenSecretManager.removeNodeKey(nodeId);
    this.nmLivelinessMonitor.register(nodeId);
    // RMNode inserted
    if (!rmContext.isWorkPreservingRecoveryEnabled()) {
        if (!request.getNMContainerStatuses().isEmpty()) {
            LOG.info("received container statuses on node manager register :" + request.getNMContainerStatuses());
            for (NMContainerStatus status : request.getNMContainerStatuses()) {
                handleNMContainerStatus(status, nodeId);
            }
        }
    }
    // Update node's labels to RM's NodeLabelManager.
    Set<String> nodeLabels = NodeLabelsUtils.convertToStringSet(request.getNodeLabels());
    if (isDistributedNodeLabelsConf && nodeLabels != null) {
        try {
            updateNodeLabelsFromNMReport(nodeLabels, nodeId);
            response.setAreNodeLabelsAcceptedByRM(true);
        } catch (IOException ex) {
            // Ensure the exception is captured in the response
            response.setDiagnosticsMessage(ex.getMessage());
            response.setAreNodeLabelsAcceptedByRM(false);
        }
    } else if (isDelegatedCentralizedNodeLabelsConf) {
        this.rmContext.getRMDelegatedNodeLabelsUpdater().updateNodeLabels(nodeId);
    }
    StringBuilder message = new StringBuilder();
    message.append("NodeManager from node ").append(host).append("(cmPort: ").append(cmPort).append(" httpPort: ");
    message.append(httpPort).append(") ").append("registered with capability: ").append(capability);
    message.append(", assigned nodeId ").append(nodeId);
    if (response.getAreNodeLabelsAcceptedByRM()) {
        message.append(", node labels { ").append(StringUtils.join(",", nodeLabels) + " } ");
    }
    LOG.info(message.toString());
    response.setNodeAction(NodeAction.NORMAL);
    response.setRMIdentifier(ResourceManager.getClusterTimeStamp());
    response.setRMVersion(YarnVersionInfo.getVersion());
    return response;
}