Examples with ResourceWeights org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights used on opensource projects

Example 1 with ResourceWeights

use of org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights in project hadoop by apache.

the class FairScheduler method getAppWeight.

public ResourceWeights getAppWeight(FSAppAttempt app) {
    try {
        readLock.lock();
        double weight = 1.0;
        if (sizeBasedWeight) {
            // Set weight based on current memory demand
            weight = Math.log1p(app.getDemand().getMemorySize()) / Math.log(2);
        }
        weight *= app.getPriority().getPriority();
        ResourceWeights resourceWeights = app.getResourceWeights();
        resourceWeights.setWeight((float) weight);
        return resourceWeights;
    } finally {
        readLock.unlock();
    }
}

Example 2 with ResourceWeights

use of org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights in project hadoop by apache.

the class AllocationFileLoaderService method loadQueue.

/**
   * Loads a queue from a queue element in the configuration file
   */
private void loadQueue(String parentName, Element element, Map<String, Resource> minQueueResources, Map<String, Resource> maxQueueResources, Map<String, Resource> maxChildQueueResources, Map<String, Integer> queueMaxApps, Map<String, Integer> userMaxApps, Map<String, Float> queueMaxAMShares, Map<String, ResourceWeights> queueWeights, Map<String, SchedulingPolicy> queuePolicies, Map<String, Long> minSharePreemptionTimeouts, Map<String, Long> fairSharePreemptionTimeouts, Map<String, Float> fairSharePreemptionThresholds, Map<String, Map<AccessType, AccessControlList>> queueAcls, Map<String, Map<ReservationACL, AccessControlList>> resAcls, Map<FSQueueType, Set<String>> configuredQueues, Set<String> reservableQueues, Set<String> nonPreemptableQueues) throws AllocationConfigurationException {
    String queueName = CharMatcher.WHITESPACE.trimFrom(element.getAttribute("name"));
    if (queueName.contains(".")) {
        throw new AllocationConfigurationException("Bad fair scheduler config " + "file: queue name (" + queueName + ") shouldn't contain period.");
    }
    if (queueName.isEmpty()) {
        throw new AllocationConfigurationException("Bad fair scheduler config " + "file: queue name shouldn't be empty or " + "consist only of whitespace.");
    }
    if (parentName != null) {
        queueName = parentName + "." + queueName;
    }
    Map<AccessType, AccessControlList> acls = new HashMap<>();
    Map<ReservationACL, AccessControlList> racls = new HashMap<>();
    NodeList fields = element.getChildNodes();
    boolean isLeaf = true;
    boolean isReservable = false;
    for (int j = 0; j < fields.getLength(); j++) {
        Node fieldNode = fields.item(j);
        if (!(fieldNode instanceof Element))
            continue;
        Element field = (Element) fieldNode;
        if ("minResources".equals(field.getTagName())) {
            String text = ((Text) field.getFirstChild()).getData().trim();
            Resource val = FairSchedulerConfiguration.parseResourceConfigValue(text);
            minQueueResources.put(queueName, val);
        } else if ("maxResources".equals(field.getTagName())) {
            String text = ((Text) field.getFirstChild()).getData().trim();
            Resource val = FairSchedulerConfiguration.parseResourceConfigValue(text);
            maxQueueResources.put(queueName, val);
        } else if ("maxChildResources".equals(field.getTagName())) {
            String text = ((Text) field.getFirstChild()).getData().trim();
            Resource val = FairSchedulerConfiguration.parseResourceConfigValue(text);
            maxChildQueueResources.put(queueName, val);
        } else if ("maxRunningApps".equals(field.getTagName())) {
            String text = ((Text) field.getFirstChild()).getData().trim();
            int val = Integer.parseInt(text);
            queueMaxApps.put(queueName, val);
        } else if ("maxAMShare".equals(field.getTagName())) {
            String text = ((Text) field.getFirstChild()).getData().trim();
            float val = Float.parseFloat(text);
            val = Math.min(val, 1.0f);
            queueMaxAMShares.put(queueName, val);
        } else if ("weight".equals(field.getTagName())) {
            String text = ((Text) field.getFirstChild()).getData().trim();
            double val = Double.parseDouble(text);
            queueWeights.put(queueName, new ResourceWeights((float) val));
        } else if ("minSharePreemptionTimeout".equals(field.getTagName())) {
            String text = ((Text) field.getFirstChild()).getData().trim();
            long val = Long.parseLong(text) * 1000L;
            minSharePreemptionTimeouts.put(queueName, val);
        } else if ("fairSharePreemptionTimeout".equals(field.getTagName())) {
            String text = ((Text) field.getFirstChild()).getData().trim();
            long val = Long.parseLong(text) * 1000L;
            fairSharePreemptionTimeouts.put(queueName, val);
        } else if ("fairSharePreemptionThreshold".equals(field.getTagName())) {
            String text = ((Text) field.getFirstChild()).getData().trim();
            float val = Float.parseFloat(text);
            val = Math.max(Math.min(val, 1.0f), 0.0f);
            fairSharePreemptionThresholds.put(queueName, val);
        } else if ("schedulingPolicy".equals(field.getTagName()) || "schedulingMode".equals(field.getTagName())) {
            String text = ((Text) field.getFirstChild()).getData().trim();
            SchedulingPolicy policy = SchedulingPolicy.parse(text);
            queuePolicies.put(queueName, policy);
        } else if ("aclSubmitApps".equals(field.getTagName())) {
            String text = ((Text) field.getFirstChild()).getData();
            acls.put(AccessType.SUBMIT_APP, new AccessControlList(text));
        } else if ("aclAdministerApps".equals(field.getTagName())) {
            String text = ((Text) field.getFirstChild()).getData();
            acls.put(AccessType.ADMINISTER_QUEUE, new AccessControlList(text));
        } else if ("aclAdministerReservations".equals(field.getTagName())) {
            String text = ((Text) field.getFirstChild()).getData();
            racls.put(ReservationACL.ADMINISTER_RESERVATIONS, new AccessControlList(text));
        } else if ("aclListReservations".equals(field.getTagName())) {
            String text = ((Text) field.getFirstChild()).getData();
            racls.put(ReservationACL.LIST_RESERVATIONS, new AccessControlList(text));
        } else if ("aclSubmitReservations".equals(field.getTagName())) {
            String text = ((Text) field.getFirstChild()).getData();
            racls.put(ReservationACL.SUBMIT_RESERVATIONS, new AccessControlList(text));
        } else if ("reservation".equals(field.getTagName())) {
            isReservable = true;
            reservableQueues.add(queueName);
            configuredQueues.get(FSQueueType.PARENT).add(queueName);
        } else if ("allowPreemptionFrom".equals(field.getTagName())) {
            String text = ((Text) field.getFirstChild()).getData().trim();
            if (!Boolean.parseBoolean(text)) {
                nonPreemptableQueues.add(queueName);
            }
        } else if ("queue".endsWith(field.getTagName()) || "pool".equals(field.getTagName())) {
            loadQueue(queueName, field, minQueueResources, maxQueueResources, maxChildQueueResources, queueMaxApps, userMaxApps, queueMaxAMShares, queueWeights, queuePolicies, minSharePreemptionTimeouts, fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls, resAcls, configuredQueues, reservableQueues, nonPreemptableQueues);
            isLeaf = false;
        }
    }
    // then store it as a parent queue
    if (isLeaf && !"parent".equals(element.getAttribute("type"))) {
        configuredQueues.get(FSQueueType.LEAF).add(queueName);
    } else {
        if (isReservable) {
            throw new AllocationConfigurationException("The configuration settings" + " for " + queueName + " are invalid. A queue element that " + "contains child queue elements or that has the type='parent' " + "attribute cannot also include a reservation element.");
        }
        configuredQueues.get(FSQueueType.PARENT).add(queueName);
    }
    // The root queue defaults to all access
    for (QueueACL acl : QueueACL.values()) {
        AccessType accessType = SchedulerUtils.toAccessType(acl);
        if (acls.get(accessType) == null) {
            AccessControlList defaultAcl = queueName.equals(ROOT) ? EVERYBODY_ACL : NOBODY_ACL;
            acls.put(accessType, defaultAcl);
        }
    }
    queueAcls.put(queueName, acls);
    resAcls.put(queueName, racls);
    if (maxQueueResources.containsKey(queueName) && minQueueResources.containsKey(queueName) && !Resources.fitsIn(minQueueResources.get(queueName), maxQueueResources.get(queueName))) {
        LOG.warn(String.format("Queue %s has max resources %s less than " + "min resources %s", queueName, maxQueueResources.get(queueName), minQueueResources.get(queueName)));
    }
}

Example 3 with ResourceWeights

use of org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights in project hadoop by apache.

the class TestDominantResourceFairnessPolicy method testCalculateShares.

@Test
public void testCalculateShares() {
    Resource used = Resources.createResource(10, 5);
    Resource capacity = Resources.createResource(100, 10);
    ResourceType[] resourceOrder = new ResourceType[2];
    ResourceWeights shares = new ResourceWeights();
    DominantResourceFairnessPolicy.DominantResourceFairnessComparator comparator = new DominantResourceFairnessPolicy.DominantResourceFairnessComparator();
    comparator.calculateShares(used, capacity, shares, resourceOrder, ResourceWeights.NEUTRAL);
    assertEquals(.1, shares.getWeight(ResourceType.MEMORY), .00001);
    assertEquals(.5, shares.getWeight(ResourceType.CPU), .00001);
    assertEquals(ResourceType.CPU, resourceOrder[0]);
    assertEquals(ResourceType.MEMORY, resourceOrder[1]);
}

Example 4 with ResourceWeights

use of org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights in project hadoop by apache.

the class AllocationFileLoaderService method reloadAllocations.

/**
   * Updates the allocation list from the allocation config file. This file is
   * expected to be in the XML format specified in the design doc.
   *
   * @throws IOException if the config file cannot be read.
   * @throws AllocationConfigurationException if allocations are invalid.
   * @throws ParserConfigurationException if XML parser is misconfigured.
   * @throws SAXException if config file is malformed.
   */
public synchronized void reloadAllocations() throws IOException, ParserConfigurationException, SAXException, AllocationConfigurationException {
    if (allocFile == null) {
        reloadListener.onReload(null);
        return;
    }
    LOG.info("Loading allocation file " + allocFile);
    // Create some temporary hashmaps to hold the new allocs, and we only save
    // them in our fields if we have parsed the entire allocs file successfully.
    Map<String, Resource> minQueueResources = new HashMap<>();
    Map<String, Resource> maxQueueResources = new HashMap<>();
    Map<String, Resource> maxChildQueueResources = new HashMap<>();
    Map<String, Integer> queueMaxApps = new HashMap<>();
    Map<String, Integer> userMaxApps = new HashMap<>();
    Map<String, Float> queueMaxAMShares = new HashMap<>();
    Map<String, ResourceWeights> queueWeights = new HashMap<>();
    Map<String, SchedulingPolicy> queuePolicies = new HashMap<>();
    Map<String, Long> minSharePreemptionTimeouts = new HashMap<>();
    Map<String, Long> fairSharePreemptionTimeouts = new HashMap<>();
    Map<String, Float> fairSharePreemptionThresholds = new HashMap<>();
    Map<String, Map<AccessType, AccessControlList>> queueAcls = new HashMap<>();
    Map<String, Map<ReservationACL, AccessControlList>> reservationAcls = new HashMap<>();
    Set<String> reservableQueues = new HashSet<>();
    Set<String> nonPreemptableQueues = new HashSet<>();
    int userMaxAppsDefault = Integer.MAX_VALUE;
    int queueMaxAppsDefault = Integer.MAX_VALUE;
    Resource queueMaxResourcesDefault = Resources.unbounded();
    float queueMaxAMShareDefault = 0.5f;
    long defaultFairSharePreemptionTimeout = Long.MAX_VALUE;
    long defaultMinSharePreemptionTimeout = Long.MAX_VALUE;
    float defaultFairSharePreemptionThreshold = 0.5f;
    SchedulingPolicy defaultSchedPolicy = SchedulingPolicy.DEFAULT_POLICY;
    // Reservation global configuration knobs
    String planner = null;
    String reservationAgent = null;
    String reservationAdmissionPolicy = null;
    QueuePlacementPolicy newPlacementPolicy = null;
    // Remember all queue names so we can display them on web UI, etc.
    // configuredQueues is segregated based on whether it is a leaf queue
    // or a parent queue. This information is used for creating queues
    // and also for making queue placement decisions(QueuePlacementRule.java).
    Map<FSQueueType, Set<String>> configuredQueues = new HashMap<>();
    for (FSQueueType queueType : FSQueueType.values()) {
        configuredQueues.put(queueType, new HashSet<String>());
    }
    // Read and parse the allocations file.
    DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
    docBuilderFactory.setIgnoringComments(true);
    DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
    Document doc = builder.parse(allocFile);
    Element root = doc.getDocumentElement();
    if (!"allocations".equals(root.getTagName()))
        throw new AllocationConfigurationException("Bad fair scheduler config " + "file: top-level element not <allocations>");
    NodeList elements = root.getChildNodes();
    List<Element> queueElements = new ArrayList<Element>();
    Element placementPolicyElement = null;
    for (int i = 0; i < elements.getLength(); i++) {
        Node node = elements.item(i);
        if (node instanceof Element) {
            Element element = (Element) node;
            if ("queue".equals(element.getTagName()) || "pool".equals(element.getTagName())) {
                queueElements.add(element);
            } else if ("user".equals(element.getTagName())) {
                String userName = element.getAttribute("name");
                NodeList fields = element.getChildNodes();
                for (int j = 0; j < fields.getLength(); j++) {
                    Node fieldNode = fields.item(j);
                    if (!(fieldNode instanceof Element))
                        continue;
                    Element field = (Element) fieldNode;
                    if ("maxRunningApps".equals(field.getTagName())) {
                        String text = ((Text) field.getFirstChild()).getData().trim();
                        int val = Integer.parseInt(text);
                        userMaxApps.put(userName, val);
                    }
                }
            } else if ("queueMaxResourcesDefault".equals(element.getTagName())) {
                String text = ((Text) element.getFirstChild()).getData().trim();
                Resource val = FairSchedulerConfiguration.parseResourceConfigValue(text);
                queueMaxResourcesDefault = val;
            } else if ("userMaxAppsDefault".equals(element.getTagName())) {
                String text = ((Text) element.getFirstChild()).getData().trim();
                int val = Integer.parseInt(text);
                userMaxAppsDefault = val;
            } else if ("defaultFairSharePreemptionTimeout".equals(element.getTagName())) {
                String text = ((Text) element.getFirstChild()).getData().trim();
                long val = Long.parseLong(text) * 1000L;
                defaultFairSharePreemptionTimeout = val;
            } else if ("fairSharePreemptionTimeout".equals(element.getTagName())) {
                if (defaultFairSharePreemptionTimeout == Long.MAX_VALUE) {
                    String text = ((Text) element.getFirstChild()).getData().trim();
                    long val = Long.parseLong(text) * 1000L;
                    defaultFairSharePreemptionTimeout = val;
                }
            } else if ("defaultMinSharePreemptionTimeout".equals(element.getTagName())) {
                String text = ((Text) element.getFirstChild()).getData().trim();
                long val = Long.parseLong(text) * 1000L;
                defaultMinSharePreemptionTimeout = val;
            } else if ("defaultFairSharePreemptionThreshold".equals(element.getTagName())) {
                String text = ((Text) element.getFirstChild()).getData().trim();
                float val = Float.parseFloat(text);
                val = Math.max(Math.min(val, 1.0f), 0.0f);
                defaultFairSharePreemptionThreshold = val;
            } else if ("queueMaxAppsDefault".equals(element.getTagName())) {
                String text = ((Text) element.getFirstChild()).getData().trim();
                int val = Integer.parseInt(text);
                queueMaxAppsDefault = val;
            } else if ("queueMaxAMShareDefault".equals(element.getTagName())) {
                String text = ((Text) element.getFirstChild()).getData().trim();
                float val = Float.parseFloat(text);
                val = Math.min(val, 1.0f);
                queueMaxAMShareDefault = val;
            } else if ("defaultQueueSchedulingPolicy".equals(element.getTagName()) || "defaultQueueSchedulingMode".equals(element.getTagName())) {
                String text = ((Text) element.getFirstChild()).getData().trim();
                if (text.equalsIgnoreCase(FifoPolicy.NAME)) {
                    throw new AllocationConfigurationException("Bad fair scheduler " + "config file: defaultQueueSchedulingPolicy or " + "defaultQueueSchedulingMode can't be FIFO.");
                }
                defaultSchedPolicy = SchedulingPolicy.parse(text);
            } else if ("queuePlacementPolicy".equals(element.getTagName())) {
                placementPolicyElement = element;
            } else if ("reservation-planner".equals(element.getTagName())) {
                String text = ((Text) element.getFirstChild()).getData().trim();
                planner = text;
            } else if ("reservation-agent".equals(element.getTagName())) {
                String text = ((Text) element.getFirstChild()).getData().trim();
                reservationAgent = text;
            } else if ("reservation-policy".equals(element.getTagName())) {
                String text = ((Text) element.getFirstChild()).getData().trim();
                reservationAdmissionPolicy = text;
            } else {
                LOG.warn("Bad element in allocations file: " + element.getTagName());
            }
        }
    }
    // it's included, all other queues must be inside it.
    for (Element element : queueElements) {
        String parent = "root";
        if (element.getAttribute("name").equalsIgnoreCase("root")) {
            if (queueElements.size() > 1) {
                throw new AllocationConfigurationException("If configuring root queue," + " no other queues can be placed alongside it.");
            }
            parent = null;
        }
        loadQueue(parent, element, minQueueResources, maxQueueResources, maxChildQueueResources, queueMaxApps, userMaxApps, queueMaxAMShares, queueWeights, queuePolicies, minSharePreemptionTimeouts, fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls, reservationAcls, configuredQueues, reservableQueues, nonPreemptableQueues);
    }
    // Load placement policy and pass it configured queues
    Configuration conf = getConfig();
    if (placementPolicyElement != null) {
        newPlacementPolicy = QueuePlacementPolicy.fromXml(placementPolicyElement, configuredQueues, conf);
    } else {
        newPlacementPolicy = QueuePlacementPolicy.fromConfiguration(conf, configuredQueues);
    }
    // Set the min/fair share preemption timeout for the root queue
    if (!minSharePreemptionTimeouts.containsKey(QueueManager.ROOT_QUEUE)) {
        minSharePreemptionTimeouts.put(QueueManager.ROOT_QUEUE, defaultMinSharePreemptionTimeout);
    }
    if (!fairSharePreemptionTimeouts.containsKey(QueueManager.ROOT_QUEUE)) {
        fairSharePreemptionTimeouts.put(QueueManager.ROOT_QUEUE, defaultFairSharePreemptionTimeout);
    }
    // Set the fair share preemption threshold for the root queue
    if (!fairSharePreemptionThresholds.containsKey(QueueManager.ROOT_QUEUE)) {
        fairSharePreemptionThresholds.put(QueueManager.ROOT_QUEUE, defaultFairSharePreemptionThreshold);
    }
    ReservationQueueConfiguration globalReservationQueueConfig = new ReservationQueueConfiguration();
    if (planner != null) {
        globalReservationQueueConfig.setPlanner(planner);
    }
    if (reservationAdmissionPolicy != null) {
        globalReservationQueueConfig.setReservationAdmissionPolicy(reservationAdmissionPolicy);
    }
    if (reservationAgent != null) {
        globalReservationQueueConfig.setReservationAgent(reservationAgent);
    }
    AllocationConfiguration info = new AllocationConfiguration(minQueueResources, maxQueueResources, maxChildQueueResources, queueMaxApps, userMaxApps, queueWeights, queueMaxAMShares, userMaxAppsDefault, queueMaxAppsDefault, queueMaxResourcesDefault, queueMaxAMShareDefault, queuePolicies, defaultSchedPolicy, minSharePreemptionTimeouts, fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls, reservationAcls, newPlacementPolicy, configuredQueues, globalReservationQueueConfig, reservableQueues, nonPreemptableQueues);
    lastSuccessfulReload = clock.getTime();
    lastReloadAttemptFailed = false;
    reloadListener.onReload(info);
}