Examples with ResourceRequest org.apache.hadoop.yarn.api.records.ResourceRequest used on opensource projects

Example 36 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class Application method getResources.

public synchronized List<Container> getResources() throws IOException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("getResources begin:" + " application=" + applicationId + " #ask=" + ask.size());
        for (ResourceRequest request : ask) {
            LOG.debug("getResources:" + " application=" + applicationId + " ask-request=" + request);
        }
    }
    // Get resources from the ResourceManager
    Allocation allocation = resourceManager.getResourceScheduler().allocate(applicationAttemptId, new ArrayList<ResourceRequest>(ask), new ArrayList<ContainerId>(), null, null, new ContainerUpdates());
    if (LOG.isInfoEnabled()) {
        LOG.info("-=======" + applicationAttemptId + System.lineSeparator() + "----------" + resourceManager.getRMContext().getRMApps().get(applicationId).getRMAppAttempt(applicationAttemptId));
    }
    List<Container> containers = allocation.getContainers();
    // Clear state for next interaction with ResourceManager
    ask.clear();
    if (LOG.isDebugEnabled()) {
        LOG.debug("getResources() for " + applicationId + ":" + " ask=" + ask.size() + " recieved=" + containers.size());
    }
    return containers;
}

Example 37 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class Application method addResourceRequest.

private synchronized void addResourceRequest(SchedulerRequestKey schedulerKey, Map<String, ResourceRequest> requests, String resourceName, Resource capability) {
    ResourceRequest request = requests.get(resourceName);
    if (request == null) {
        request = org.apache.hadoop.yarn.server.utils.BuilderUtils.newResourceRequest(schedulerKey.getPriority(), resourceName, capability, 1);
        requests.put(resourceName, request);
    } else {
        request.setNumContainers(request.getNumContainers() + 1);
    }
    if (request.getNodeLabelExpression() == null) {
        request.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL);
    }
    // Note this down for next interaction with ResourceManager
    ask.remove(request);
    ask.add(org.apache.hadoop.yarn.server.utils.BuilderUtils.newResourceRequest(// clone to ensure the RM doesn't manipulate the same obj
    request));
    if (LOG.isDebugEnabled()) {
        LOG.debug("addResourceRequest: applicationId=" + applicationId.getId() + " priority=" + schedulerKey.getPriority().getPriority() + " resourceName=" + resourceName + " capability=" + capability + " numContainers=" + request.getNumContainers() + " #asks=" + ask.size());
    }
}

Example 38 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class MockAM method createReq.

public List<ResourceRequest> createReq(String[] hosts, int memory, int priority, int containers, String labelExpression, long allocationRequestId) throws Exception {
    List<ResourceRequest> reqs = new ArrayList<ResourceRequest>();
    if (hosts != null) {
        for (String host : hosts) {
            // only add host/rack request when asked host isn't ANY
            if (!host.equals(ResourceRequest.ANY)) {
                ResourceRequest hostReq = createResourceReq(host, memory, priority, containers, labelExpression);
                hostReq.setAllocationRequestId(allocationRequestId);
                reqs.add(hostReq);
                ResourceRequest rackReq = createResourceReq("/default-rack", memory, priority, containers, labelExpression);
                rackReq.setAllocationRequestId(allocationRequestId);
                reqs.add(rackReq);
            }
        }
    }
    ResourceRequest offRackReq = createResourceReq(ResourceRequest.ANY, memory, priority, containers, labelExpression);
    offRackReq.setAllocationRequestId(allocationRequestId);
    reqs.add(offRackReq);
    return reqs;
}

Example 39 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class MockRM method submitApp.

public RMApp submitApp(Resource capability, String name, String user, Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue, int maxAppAttempts, Credentials ts, String appType, boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, ApplicationId applicationId, long attemptFailuresValidityInterval, LogAggregationContext logAggregationContext, boolean cancelTokensWhenComplete, Priority priority, String amLabel, Map<ApplicationTimeoutType, Long> applicationTimeouts, ByteBuffer tokensConf) throws Exception {
    ApplicationId appId = isAppIdProvided ? applicationId : null;
    ApplicationClientProtocol client = getClientRMService();
    if (!isAppIdProvided) {
        GetNewApplicationResponse resp = client.getNewApplication(Records.newRecord(GetNewApplicationRequest.class));
        appId = resp.getApplicationId();
    }
    SubmitApplicationRequest req = Records.newRecord(SubmitApplicationRequest.class);
    ApplicationSubmissionContext sub = Records.newRecord(ApplicationSubmissionContext.class);
    sub.setKeepContainersAcrossApplicationAttempts(keepContainers);
    sub.setApplicationId(appId);
    sub.setApplicationName(name);
    sub.setMaxAppAttempts(maxAppAttempts);
    if (applicationTimeouts != null && applicationTimeouts.size() > 0) {
        sub.setApplicationTimeouts(applicationTimeouts);
    }
    if (unmanaged) {
        sub.setUnmanagedAM(true);
    }
    if (queue != null) {
        sub.setQueue(queue);
    }
    if (priority != null) {
        sub.setPriority(priority);
    }
    sub.setApplicationType(appType);
    ContainerLaunchContext clc = Records.newRecord(ContainerLaunchContext.class);
    sub.setResource(capability);
    clc.setApplicationACLs(acls);
    if (ts != null && UserGroupInformation.isSecurityEnabled()) {
        DataOutputBuffer dob = new DataOutputBuffer();
        ts.writeTokenStorageToStream(dob);
        ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
        clc.setTokens(securityTokens);
        clc.setTokensConf(tokensConf);
    }
    sub.setAMContainerSpec(clc);
    sub.setAttemptFailuresValidityInterval(attemptFailuresValidityInterval);
    if (logAggregationContext != null) {
        sub.setLogAggregationContext(logAggregationContext);
    }
    sub.setCancelTokensWhenComplete(cancelTokensWhenComplete);
    ResourceRequest amResourceRequest = ResourceRequest.newInstance(Priority.newInstance(0), ResourceRequest.ANY, capability, 1);
    if (amLabel != null && !amLabel.isEmpty()) {
        amResourceRequest.setNodeLabelExpression(amLabel.trim());
    }
    sub.setAMContainerResourceRequest(amResourceRequest);
    req.setApplicationSubmissionContext(sub);
    UserGroupInformation fakeUser = UserGroupInformation.createUserForTesting(user, new String[] { "someGroup" });
    PrivilegedExceptionAction<SubmitApplicationResponse> action = new PrivilegedExceptionAction<SubmitApplicationResponse>() {

        ApplicationClientProtocol client;

        SubmitApplicationRequest req;

        @Override
        public SubmitApplicationResponse run() throws IOException, YarnException {
            try {
                return client.submitApplication(req);
            } catch (YarnException | IOException e) {
                e.printStackTrace();
                throw e;
            }
        }

        PrivilegedExceptionAction<SubmitApplicationResponse> setClientReq(ApplicationClientProtocol client, SubmitApplicationRequest req) {
            this.client = client;
            this.req = req;
            return this;
        }
    }.setClientReq(client, req);
    fakeUser.doAs(action);
    // make sure app is immediately available after submit
    if (waitForAccepted) {
        waitForState(appId, RMAppState.ACCEPTED);
    }
    RMApp rmApp = getRMContext().getRMApps().get(appId);
    // unmanaged AM won't go to RMAppAttemptState.SCHEDULED.
    if (waitForAccepted && !unmanaged) {
        waitForState(rmApp.getCurrentAppAttempt().getAppAttemptId(), RMAppAttemptState.SCHEDULED);
    }
    return rmApp;
}

Example 40 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class MRAMSimulator method sendContainerRequest.

@Override
protected void sendContainerRequest() throws YarnException, IOException, InterruptedException {
    if (isFinished) {
        return;
    }
    // send out request
    List<ResourceRequest> ask = null;
    if (mapFinished != mapTotal) {
        // map phase
        if (!pendingMaps.isEmpty()) {
            ask = packageRequests(mergeLists(pendingMaps, scheduledMaps), PRIORITY_MAP);
            LOG.debug(MessageFormat.format("Application {0} sends out " + "request for {1} mappers.", appId, pendingMaps.size()));
            scheduledMaps.addAll(pendingMaps);
            pendingMaps.clear();
        } else if (!pendingFailedMaps.isEmpty()) {
            ask = packageRequests(mergeLists(pendingFailedMaps, scheduledMaps), PRIORITY_MAP);
            LOG.debug(MessageFormat.format("Application {0} sends out " + "requests for {1} failed mappers.", appId, pendingFailedMaps.size()));
            scheduledMaps.addAll(pendingFailedMaps);
            pendingFailedMaps.clear();
        }
    } else if (reduceFinished != reduceTotal) {
        // reduce phase
        if (!pendingReduces.isEmpty()) {
            ask = packageRequests(mergeLists(pendingReduces, scheduledReduces), PRIORITY_REDUCE);
            LOG.debug(MessageFormat.format("Application {0} sends out " + "requests for {1} reducers.", appId, pendingReduces.size()));
            scheduledReduces.addAll(pendingReduces);
            pendingReduces.clear();
        } else if (!pendingFailedReduces.isEmpty()) {
            ask = packageRequests(mergeLists(pendingFailedReduces, scheduledReduces), PRIORITY_REDUCE);
            LOG.debug(MessageFormat.format("Application {0} sends out " + "request for {1} failed reducers.", appId, pendingFailedReduces.size()));
            scheduledReduces.addAll(pendingFailedReduces);
            pendingFailedReduces.clear();
        }
    }
    if (ask == null) {
        ask = new ArrayList<>();
    }
    final AllocateRequest request = createAllocateRequest(ask);
    if (totalContainers == 0) {
        request.setProgress(1.0f);
    } else {
        request.setProgress((float) finishedContainers / totalContainers);
    }
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(appAttemptId.toString());
    Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps().get(appAttemptId.getApplicationId()).getRMAppAttempt(appAttemptId).getAMRMToken();
    ugi.addTokenIdentifier(token.decodeIdentifier());
    AllocateResponse response = ugi.doAs(new PrivilegedExceptionAction<AllocateResponse>() {

        @Override
        public AllocateResponse run() throws Exception {
            return rm.getApplicationMasterService().allocate(request);
        }
    });
    if (response != null) {
        responseQueue.put(response);
    }
}