Examples with ResourceRequest org.apache.hadoop.yarn.api.records.ResourceRequest used on opensource projects

Example 71 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project weave by continuuity.

the class AMRMClientImpl method addResourceRequest.

private void addResourceRequest(Priority priority, String resourceName, Resource capability, int containerCount) {
    Map<String, Map<Resource, ResourceRequest>> remoteRequests = this.remoteRequestsTable.get(priority);
    if (remoteRequests == null) {
        remoteRequests = new HashMap<String, Map<Resource, ResourceRequest>>();
        this.remoteRequestsTable.put(priority, remoteRequests);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Added priority=" + priority);
        }
    }
    Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
    if (reqMap == null) {
        reqMap = new HashMap<Resource, ResourceRequest>();
        remoteRequests.put(resourceName, reqMap);
    }
    ResourceRequest remoteRequest = reqMap.get(capability);
    if (remoteRequest == null) {
        remoteRequest = BuilderUtils.newResourceRequest(priority, resourceName, capability, 0);
        reqMap.put(capability, remoteRequest);
    }
    remoteRequest.setNumContainers(remoteRequest.getNumContainers() + containerCount);
    // Note this down for next interaction with ResourceManager
    addResourceRequestToAsk(remoteRequest);
    if (LOG.isDebugEnabled()) {
        LOG.debug("addResourceRequest:" + " applicationId=" + appAttemptId + " priority=" + priority.getPriority() + " resourceName=" + resourceName + " numContainers=" + remoteRequest.getNumContainers() + " #asks=" + ask.size());
    }
}

Example 72 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project weave by continuuity.

the class AMRMClientImpl method allocate.

@Override
public AllocationResponse allocate(float progressIndicator) throws YarnRemoteException {
    AllocateResponse allocateResponse = null;
    ArrayList<ResourceRequest> askList = null;
    ArrayList<ContainerId> releaseList = null;
    AllocateRequest allocateRequest = null;
    try {
        synchronized (this) {
            askList = new ArrayList<ResourceRequest>(ask);
            releaseList = new ArrayList<ContainerId>(release);
            // optimistically clear this collection assuming no RPC failure
            ask.clear();
            release.clear();
            allocateRequest = BuilderUtils.newAllocateRequest(appAttemptId, lastResponseId, progressIndicator, askList, releaseList);
        }
        allocateResponse = rmClient.allocate(allocateRequest);
        AllocationResponse response = AllocationResponses.create(allocateResponse);
        synchronized (this) {
            // update these on successful RPC
            clusterNodeCount = allocateResponse.getNumClusterNodes();
            lastResponseId = response.getResponseId();
            clusterAvailableResources = response.getAvailableResources();
        }
        return response;
    } finally {
        // TODO how to differentiate remote YARN exception vs error in RPC
        if (allocateResponse == null) {
            // Preserve ask and release for next call to allocate()
            synchronized (this) {
                release.addAll(releaseList);
                // synchronized block at the beginning of this method.
                for (ResourceRequest oldAsk : askList) {
                    if (!ask.contains(oldAsk)) {
                        ask.add(oldAsk);
                    }
                }
            }
        }
    }
}

Example 73 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class FifoAppAttempt method allocate.

public RMContainer allocate(NodeType type, FiCaSchedulerNode node, SchedulerRequestKey schedulerKey, Container container) {
    try {
        writeLock.lock();
        if (isStopped) {
            return null;
        }
        // request without locking the scheduler, hence we need to check
        if (getOutstandingAsksCount(schedulerKey) <= 0) {
            return null;
        }
        // Create RMContainer
        RMContainer rmContainer = new RMContainerImpl(container, schedulerKey, this.getApplicationAttemptId(), node.getNodeID(), appSchedulingInfo.getUser(), this.rmContext, node.getPartition());
        ((RMContainerImpl) rmContainer).setQueueName(this.getQueueName());
        updateAMContainerDiagnostics(AMState.ASSIGNED, null);
        // Add it to allContainers list.
        addToNewlyAllocatedContainers(node, rmContainer);
        ContainerId containerId = container.getId();
        liveContainers.put(containerId, rmContainer);
        // Update consumption and track allocations
        List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(type, node, schedulerKey, container);
        attemptResourceUsage.incUsed(node.getPartition(), container.getResource());
        // Update resource requests related to "request" and store in RMContainer
        ((RMContainerImpl) rmContainer).setResourceRequests(resourceRequestList);
        // Inform the container
        rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.START));
        if (LOG.isDebugEnabled()) {
            LOG.debug("allocate: applicationAttemptId=" + containerId.getApplicationAttemptId() + " container=" + containerId + " host=" + container.getNodeId().getHost() + " type=" + type);
        }
        RMAuditLogger.logSuccess(getUser(), RMAuditLogger.AuditConstants.ALLOC_CONTAINER, "SchedulerApp", getApplicationId(), containerId, container.getResource());
        return rmContainer;
    } finally {
        writeLock.unlock();
    }
}

Example 74 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class FiCaSchedulerApp method apply.

public void apply(Resource cluster, ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> request) {
    boolean reReservation = false;
    try {
        writeLock.lock();
        // If we allocated something
        if (request.anythingAllocatedOrReserved()) {
            ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode> allocation = request.getFirstAllocatedOrReservedContainer();
            SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode> schedulerContainer = allocation.getAllocatedOrReservedContainer();
            RMContainer rmContainer = schedulerContainer.getRmContainer();
            reReservation = (!schedulerContainer.isAllocated()) && (rmContainer.getState() == RMContainerState.RESERVED);
            // Or re-reservation
            if (rmContainer.getContainer().getId() == null) {
                rmContainer.setContainerId(BuilderUtils.newContainerId(getApplicationAttemptId(), getNewContainerId()));
            }
            ContainerId containerId = rmContainer.getContainerId();
            if (schedulerContainer.isAllocated()) {
                // Unreserve it first
                if (allocation.getAllocateFromReservedContainer() != null) {
                    RMContainer reservedContainer = allocation.getAllocateFromReservedContainer().getRmContainer();
                    // Handling container allocation
                    // Did we previously reserve containers at this 'priority'?
                    unreserve(schedulerContainer.getSchedulerRequestKey(), schedulerContainer.getSchedulerNode(), reservedContainer);
                }
                // Allocate a new container
                addToNewlyAllocatedContainers(schedulerContainer.getSchedulerNode(), rmContainer);
                liveContainers.put(containerId, rmContainer);
                // Deduct pending resource requests
                List<ResourceRequest> requests = appSchedulingInfo.allocate(allocation.getAllocationLocalityType(), schedulerContainer.getSchedulerNode(), schedulerContainer.getSchedulerRequestKey(), schedulerContainer.getRmContainer().getContainer());
                ((RMContainerImpl) rmContainer).setResourceRequests(requests);
                attemptResourceUsage.incUsed(schedulerContainer.getNodePartition(), allocation.getAllocatedOrReservedResource());
                rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.START));
                // Inform the node
                schedulerContainer.getSchedulerNode().allocateContainer(rmContainer);
                // update locality statistics,
                incNumAllocatedContainers(allocation.getAllocationLocalityType(), allocation.getRequestLocalityType());
                if (LOG.isDebugEnabled()) {
                    LOG.debug("allocate: applicationAttemptId=" + containerId.getApplicationAttemptId() + " container=" + containerId + " host=" + rmContainer.getAllocatedNode().getHost() + " type=" + allocation.getAllocationLocalityType());
                }
                RMAuditLogger.logSuccess(getUser(), AuditConstants.ALLOC_CONTAINER, "SchedulerApp", getApplicationId(), containerId, allocation.getAllocatedOrReservedResource());
            } else {
                // If the rmContainer's state is already updated to RESERVED, this is
                // a reReservation
                reserve(schedulerContainer.getSchedulerRequestKey(), schedulerContainer.getSchedulerNode(), schedulerContainer.getRmContainer(), schedulerContainer.getRmContainer().getContainer(), reReservation);
            }
        }
    } finally {
        writeLock.unlock();
    }
    // Don't bother CS leaf queue if it is a re-reservation
    if (!reReservation) {
        getCSLeafQueue().apply(cluster, request);
    }
}

Example 75 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class MockAM method createResourceReq.

public ResourceRequest createResourceReq(String resource, int memory, int priority, int containers, String labelExpression, ExecutionTypeRequest executionTypeRequest) throws Exception {
    ResourceRequest req = Records.newRecord(ResourceRequest.class);
    req.setResourceName(resource);
    req.setNumContainers(containers);
    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(priority);
    req.setPriority(pri);
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemorySize(memory);
    req.setCapability(capability);
    if (labelExpression != null) {
        req.setNodeLabelExpression(labelExpression);
    }
    req.setExecutionTypeRequest(executionTypeRequest);
    return req;
}