Search in sources :

Example 1 with ReservationRequestInterpreter

use of org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter in project hadoop by apache.

the class RMWebServices method createReservationSubmissionRequest.

private ReservationSubmissionRequest createReservationSubmissionRequest(ReservationSubmissionRequestInfo resContext) throws IOException {
    // defending against a couple of common submission format problems
    if (resContext == null) {
        throw new BadRequestException("Input ReservationSubmissionContext should not be null");
    }
    ReservationDefinitionInfo resInfo = resContext.getReservationDefinition();
    if (resInfo == null) {
        throw new BadRequestException("Input ReservationDefinition should not be null");
    }
    ReservationRequestsInfo resReqsInfo = resInfo.getReservationRequests();
    if (resReqsInfo == null || resReqsInfo.getReservationRequest() == null || resReqsInfo.getReservationRequest().size() == 0) {
        throw new BadRequestException("The ReservationDefinition should" + " contain at least one ReservationRequest");
    }
    ReservationRequestInterpreter[] values = ReservationRequestInterpreter.values();
    ReservationRequestInterpreter resInt = values[resReqsInfo.getReservationRequestsInterpreter()];
    List<ReservationRequest> list = new ArrayList<ReservationRequest>();
    for (ReservationRequestInfo resReqInfo : resReqsInfo.getReservationRequest()) {
        ResourceInfo rInfo = resReqInfo.getCapability();
        Resource capability = Resource.newInstance(rInfo.getMemorySize(), rInfo.getvCores());
        int numContainers = resReqInfo.getNumContainers();
        int minConcurrency = resReqInfo.getMinConcurrency();
        long duration = resReqInfo.getDuration();
        ReservationRequest rr = ReservationRequest.newInstance(capability, numContainers, minConcurrency, duration);
        list.add(rr);
    }
    ReservationRequests reqs = ReservationRequests.newInstance(list, resInt);
    ReservationDefinition rDef = ReservationDefinition.newInstance(resInfo.getArrival(), resInfo.getDeadline(), reqs, resInfo.getReservationName());
    ReservationId reservationId = ReservationId.parseReservationId(resContext.getReservationId());
    ReservationSubmissionRequest request = ReservationSubmissionRequest.newInstance(rDef, resContext.getQueue(), reservationId);
    return request;
}
Also used : ReservationRequestsInfo(org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestsInfo) ResourceInfo(org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo) LocalResourceInfo(org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo) ReservationRequestInterpreter(org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter) ReservationRequest(org.apache.hadoop.yarn.api.records.ReservationRequest) GetNewReservationRequest(org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest) ReservationDefinition(org.apache.hadoop.yarn.api.records.ReservationDefinition) ArrayList(java.util.ArrayList) Resource(org.apache.hadoop.yarn.api.records.Resource) LocalResource(org.apache.hadoop.yarn.api.records.LocalResource) ReservationSubmissionRequest(org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest) ReservationRequestInfo(org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestInfo) ReservationId(org.apache.hadoop.yarn.api.records.ReservationId) ReservationDefinitionInfo(org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDefinitionInfo) ReservationRequests(org.apache.hadoop.yarn.api.records.ReservationRequests) BadRequestException(org.apache.hadoop.yarn.webapp.BadRequestException)

Example 2 with ReservationRequestInterpreter

use of org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter in project hadoop by apache.

the class RMWebServices method createReservationUpdateRequest.

private ReservationUpdateRequest createReservationUpdateRequest(ReservationUpdateRequestInfo resContext) throws IOException {
    // defending against a couple of common submission format problems
    if (resContext == null) {
        throw new BadRequestException("Input ReservationSubmissionContext should not be null");
    }
    ReservationDefinitionInfo resInfo = resContext.getReservationDefinition();
    if (resInfo == null) {
        throw new BadRequestException("Input ReservationDefinition should not be null");
    }
    ReservationRequestsInfo resReqsInfo = resInfo.getReservationRequests();
    if (resReqsInfo == null || resReqsInfo.getReservationRequest() == null || resReqsInfo.getReservationRequest().size() == 0) {
        throw new BadRequestException("The ReservationDefinition should" + " contain at least one ReservationRequest");
    }
    if (resContext.getReservationId() == null) {
        throw new BadRequestException("Update operations must specify an existing ReservaitonId");
    }
    ReservationRequestInterpreter[] values = ReservationRequestInterpreter.values();
    ReservationRequestInterpreter resInt = values[resReqsInfo.getReservationRequestsInterpreter()];
    List<ReservationRequest> list = new ArrayList<ReservationRequest>();
    for (ReservationRequestInfo resReqInfo : resReqsInfo.getReservationRequest()) {
        ResourceInfo rInfo = resReqInfo.getCapability();
        Resource capability = Resource.newInstance(rInfo.getMemorySize(), rInfo.getvCores());
        int numContainers = resReqInfo.getNumContainers();
        int minConcurrency = resReqInfo.getMinConcurrency();
        long duration = resReqInfo.getDuration();
        ReservationRequest rr = ReservationRequest.newInstance(capability, numContainers, minConcurrency, duration);
        list.add(rr);
    }
    ReservationRequests reqs = ReservationRequests.newInstance(list, resInt);
    ReservationDefinition rDef = ReservationDefinition.newInstance(resInfo.getArrival(), resInfo.getDeadline(), reqs, resInfo.getReservationName());
    ReservationUpdateRequest request = ReservationUpdateRequest.newInstance(rDef, ReservationId.parseReservationId(resContext.getReservationId()));
    return request;
}
Also used : ReservationRequestsInfo(org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestsInfo) ResourceInfo(org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo) LocalResourceInfo(org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo) ReservationUpdateRequest(org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest) ReservationRequestInterpreter(org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter) ReservationRequest(org.apache.hadoop.yarn.api.records.ReservationRequest) GetNewReservationRequest(org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest) ReservationDefinition(org.apache.hadoop.yarn.api.records.ReservationDefinition) ArrayList(java.util.ArrayList) Resource(org.apache.hadoop.yarn.api.records.Resource) LocalResource(org.apache.hadoop.yarn.api.records.LocalResource) ReservationRequestInfo(org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestInfo) ReservationDefinitionInfo(org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDefinitionInfo) ReservationRequests(org.apache.hadoop.yarn.api.records.ReservationRequests) BadRequestException(org.apache.hadoop.yarn.webapp.BadRequestException)

Example 3 with ReservationRequestInterpreter

use of org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter in project hadoop by apache.

the class ReservationSystemTestUtil method generateRandomRR.

public static ReservationDefinition generateRandomRR(Random rand, long i) {
    rand.setSeed(i);
    long now = System.currentTimeMillis();
    // start time at random in the next 12 hours
    long arrival = rand.nextInt(12 * 3600 * 1000);
    // deadline at random in the next day
    long deadline = arrival + rand.nextInt(24 * 3600 * 1000);
    // create a request with a single atomic ask
    ReservationDefinition rr = new ReservationDefinitionPBImpl();
    rr.setArrival(now + arrival);
    rr.setDeadline(now + deadline);
    int gang = 1 + rand.nextInt(9);
    int par = (rand.nextInt(1000) + 1) * gang;
    // random duration within 2h
    long dur = rand.nextInt(2 * 3600 * 1000);
    ReservationRequest r = ReservationRequest.newInstance(Resource.newInstance(1024, 1), par, gang, dur);
    ReservationRequests reqs = new ReservationRequestsPBImpl();
    reqs.setReservationResources(Collections.singletonList(r));
    rand.nextInt(3);
    ReservationRequestInterpreter[] type = ReservationRequestInterpreter.values();
    reqs.setInterpreter(type[rand.nextInt(type.length)]);
    rr.setReservationRequests(reqs);
    return rr;
}
Also used : ReservationRequestsPBImpl(org.apache.hadoop.yarn.api.records.impl.pb.ReservationRequestsPBImpl) ReservationRequestInterpreter(org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter) ReservationDefinition(org.apache.hadoop.yarn.api.records.ReservationDefinition) ReservationRequest(org.apache.hadoop.yarn.api.records.ReservationRequest) ReservationDefinitionPBImpl(org.apache.hadoop.yarn.api.records.impl.pb.ReservationDefinitionPBImpl) ReservationRequests(org.apache.hadoop.yarn.api.records.ReservationRequests)

Example 4 with ReservationRequestInterpreter

use of org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter in project hadoop by apache.

the class ReservationInputValidator method validateReservationDefinition.

private void validateReservationDefinition(ReservationId reservationId, ReservationDefinition contract, Plan plan, String auditConstant) throws YarnException {
    String message = "";
    // check if deadline is in the past
    if (contract == null) {
        message = "Missing reservation definition." + " Please try again by specifying a reservation definition.";
        RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
        throw RPCUtil.getRemoteException(message);
    }
    if (contract.getDeadline() <= clock.getTime()) {
        message = "The specified deadline: " + contract.getDeadline() + " is the past. Please try again with deadline in the future.";
        RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
        throw RPCUtil.getRemoteException(message);
    }
    // Check if at least one RR has been specified
    ReservationRequests resReqs = contract.getReservationRequests();
    if (resReqs == null) {
        message = "No resources have been specified to reserve." + "Please try again by specifying the resources to reserve.";
        RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
        throw RPCUtil.getRemoteException(message);
    }
    List<ReservationRequest> resReq = resReqs.getReservationResources();
    if (resReq == null || resReq.isEmpty()) {
        message = "No resources have been specified to reserve." + " Please try again by specifying the resources to reserve.";
        RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
        throw RPCUtil.getRemoteException(message);
    }
    // compute minimum duration and max gang size
    long minDuration = 0;
    Resource maxGangSize = Resource.newInstance(0, 0);
    ReservationRequestInterpreter type = contract.getReservationRequests().getInterpreter();
    for (ReservationRequest rr : resReq) {
        if (type == ReservationRequestInterpreter.R_ALL || type == ReservationRequestInterpreter.R_ANY) {
            minDuration = Math.max(minDuration, rr.getDuration());
        } else {
            minDuration += rr.getDuration();
        }
        maxGangSize = Resources.max(plan.getResourceCalculator(), plan.getTotalCapacity(), maxGangSize, Resources.multiply(rr.getCapability(), rr.getConcurrency()));
    }
    // verify the allocation is possible (skip for ANY)
    if (contract.getDeadline() - contract.getArrival() < minDuration && type != ReservationRequestInterpreter.R_ANY) {
        message = "The time difference (" + (contract.getDeadline() - contract.getArrival()) + ") between arrival (" + contract.getArrival() + ") " + "and deadline (" + contract.getDeadline() + ") must " + " be greater or equal to the minimum resource duration (" + minDuration + ")";
        RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
        throw RPCUtil.getRemoteException(message);
    }
    // capacity (skip for ANY)
    if (Resources.greaterThan(plan.getResourceCalculator(), plan.getTotalCapacity(), maxGangSize, plan.getTotalCapacity()) && type != ReservationRequestInterpreter.R_ANY) {
        message = "The size of the largest gang in the reservation definition (" + maxGangSize + ") exceed the capacity available (" + plan.getTotalCapacity() + " )";
        RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
        throw RPCUtil.getRemoteException(message);
    }
    // check that the recurrence is a positive long value.
    String recurrenceExpression = contract.getRecurrenceExpression();
    try {
        Long recurrence = Long.parseLong(recurrenceExpression);
        if (recurrence < 0) {
            message = "Negative Period : " + recurrenceExpression + ". Please try" + " again with a non-negative long value as period";
            throw RPCUtil.getRemoteException(message);
        }
    } catch (NumberFormatException e) {
        message = "Invalid period " + recurrenceExpression + ". Please try" + " again with a non-negative long value as period";
        throw RPCUtil.getRemoteException(message);
    }
}
Also used : ReservationRequestInterpreter(org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter) ReservationRequest(org.apache.hadoop.yarn.api.records.ReservationRequest) ReservationRequests(org.apache.hadoop.yarn.api.records.ReservationRequests) Resource(org.apache.hadoop.yarn.api.records.Resource)

Aggregations

ReservationRequest (org.apache.hadoop.yarn.api.records.ReservationRequest)4 ReservationRequestInterpreter (org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter)4 ReservationRequests (org.apache.hadoop.yarn.api.records.ReservationRequests)4 ReservationDefinition (org.apache.hadoop.yarn.api.records.ReservationDefinition)3 Resource (org.apache.hadoop.yarn.api.records.Resource)3 ArrayList (java.util.ArrayList)2 GetNewReservationRequest (org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest)2 LocalResource (org.apache.hadoop.yarn.api.records.LocalResource)2 LocalResourceInfo (org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo)2 ReservationDefinitionInfo (org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDefinitionInfo)2 ReservationRequestInfo (org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestInfo)2 ReservationRequestsInfo (org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestsInfo)2 ResourceInfo (org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo)2 BadRequestException (org.apache.hadoop.yarn.webapp.BadRequestException)2 ReservationSubmissionRequest (org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest)1 ReservationUpdateRequest (org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest)1 ReservationId (org.apache.hadoop.yarn.api.records.ReservationId)1 ReservationDefinitionPBImpl (org.apache.hadoop.yarn.api.records.impl.pb.ReservationDefinitionPBImpl)1 ReservationRequestsPBImpl (org.apache.hadoop.yarn.api.records.impl.pb.ReservationRequestsPBImpl)1