Examples with ValueRange com.microsoft.frameworklauncher.common.model.ValueRange used on opensource projects

Example 1 with ValueRange

use of com.microsoft.frameworklauncher.common.model.ValueRange in project pai by Microsoft.

the class SelectionManager method select.

@VisibleForTesting
public synchronized SelectionResult select(ResourceDescriptor requestResource, String requestNodeLabel, String requestNodeGpuType, int startStatesTaskCount, List<ValueRange> reusePorts, Map<String, NodeConfiguration> configuredNodes) throws NotAvailableException {
    LOGGER.logInfo("select: Request: Resource: [%s], NodeLabel: [%s], NodeGpuType: [%s], StartStatesTaskCount: [%d], ReusePorts: [%s]", requestResource, requestNodeLabel, requestNodeGpuType, startStatesTaskCount, ValueRangeUtils.toString(reusePorts));
    initFilteredNodes();
    filterNodesByNodeLabel(requestNodeLabel);
    filterNodesByGpuType(configuredNodes, requestNodeGpuType);
    if (!conf.getAmAllowNoneGpuJobOnGpuNode()) {
        int jobTotalRequestGpu = requestManager.getTotalGpuCount();
        filterNodesForNoneGpuJob(jobTotalRequestGpu);
    }
    ResourceDescriptor optimizedRequestResource = YamlUtils.deepCopy(requestResource, ResourceDescriptor.class);
    if (ValueRangeUtils.getValueNumber(reusePorts) > 0) {
        LOGGER.logInfo("select: reuse pre-selected ports: [%s]", ValueRangeUtils.toString(reusePorts));
        optimizedRequestResource.setPortRanges(reusePorts);
    }
    filterNodesByResource(optimizedRequestResource, conf.getAmSkipLocalTriedResource());
    filterNodesByRackSelectionPolicy(optimizedRequestResource, startStatesTaskCount);
    if (filteredNodes.size() < 1) {
        // Don't have candidate nodes for this request.
        if (requestNodeGpuType != null || requestResource.getPortNumber() > 0) {
            // GpuType and port relax are not support in yarn, If gpuType or portNumber is specified, abort this request and try later.
            throw new NotAvailableException(String.format("Don't have enough nodes to meet request: optimizedRequestResource: [%s], NodeGpuType: [%s], NodeLabel: [%s]", optimizedRequestResource, requestNodeGpuType, requestNodeLabel));
        }
    }
    SelectionResult selectionResult = selectNodes(optimizedRequestResource, startStatesTaskCount);
    List<ValueRange> portRanges = selectPorts(selectionResult, optimizedRequestResource);
    optimizedRequestResource.setPortRanges(portRanges);
    selectionResult.setOptimizedResource(optimizedRequestResource);
    return selectionResult;
}

Example 2 with ValueRange

use of com.microsoft.frameworklauncher.common.model.ValueRange in project pai by Microsoft.

the class ValueRangeUtils method getSubRange.

/*
    get a random subRange list from the available range list, all the values in the subRange are bigger than baseValue.
   */
public static List<ValueRange> getSubRange(List<ValueRange> availableRange, int requestNumber, int baseValue) {
    List<ValueRange> resultList = new ArrayList<ValueRange>();
    Random random = new Random();
    // Pick a random number from 0 to the max value;
    int maxValue = availableRange.get(availableRange.size() - 1).getEnd();
    int randomBase = random.nextInt(maxValue) + 1;
    // number when randomBase reduce to 0, return null.
    while (randomBase > 0) {
        resultList.clear();
        int needNumber = requestNumber;
        randomBase = randomBase / 2;
        int newbaseValue = baseValue + randomBase;
        for (ValueRange range : availableRange) {
            if (range.getEnd() < newbaseValue) {
                continue;
            }
            int start = Math.max(range.getBegin(), newbaseValue);
            if ((range.getEnd() - start + 1) >= needNumber) {
                resultList.add(ValueRange.newInstance(start, start + needNumber - 1));
                return resultList;
            } else {
                resultList.add(ValueRange.newInstance(start, range.getEnd()));
                needNumber -= (range.getEnd() - start + 1);
            }
        }
    }
    return null;
}

Example 3 with ValueRange

use of com.microsoft.frameworklauncher.common.model.ValueRange in project pai by Microsoft.

the class ValueRangeUtils method getValueNumber.

/*
    count the value number in a  range list.
   */
public static int getValueNumber(List<ValueRange> rangeList) {
    if (rangeList == null || rangeList.size() == 0) {
        return 0;
    }
    List<ValueRange> newRangeList = coalesceRangeList(rangeList);
    int valueCount = 0;
    for (ValueRange range : newRangeList) {
        valueCount += (range.getEnd() - range.getBegin() + 1);
    }
    return valueCount;
}

Example 4 with ValueRange

use of com.microsoft.frameworklauncher.common.model.ValueRange in project pai by Microsoft.

the class ValueRangeUtils method subtractRange.

/*
    delete the overlap part from leftRange.
   */
public static List<ValueRange> subtractRange(List<ValueRange> leftRange, List<ValueRange> rightRange) {
    if (leftRange == null || rightRange == null) {
        return leftRange;
    }
    List<ValueRange> result = coalesceRangeList(leftRange);
    List<ValueRange> rightList = coalesceRangeList(rightRange);
    int i = 0;
    int j = 0;
    while (i < result.size() && j < rightList.size()) {
        ValueRange left = result.get(i);
        ValueRange right = rightList.get(j);
        // 1. no overlap, right is bigger than left
        if (left.getEnd() < right.getBegin()) {
            i++;
        // 2. no overlap, left is bigger than right
        } else if (right.getEnd() < left.getBegin()) {
            j++;
        // 3. has overlap, left is less than right
        } else {
            if (left.getBegin() < right.getBegin()) {
                // 3.1 Left start earlier than right, cut at the right begin;
                if (left.getEnd() <= right.getEnd()) {
                    // 3.1.1 Left end earlier than right, do nothing, try next left;
                    left.setEnd(right.getBegin() - 1);
                    i++;
                } else {
                    // 3.1.2 Left end later than right, create a new range in left;
                    ValueRange newRange = ValueRange.newInstance(right.getEnd() + 1, left.getEnd());
                    result.add(i + 1, newRange);
                    left.setEnd(right.getBegin() - 1);
                    j++;
                }
            } else {
                // 3.2 left start later than right
                if (left.getEnd() <= right.getEnd()) {
                    // 3.2.1 left end earlier than right, just remove the left
                    result.remove(i);
                } else {
                    // 3.2.2 left end later than right, just remove left
                    left.setBegin(right.getEnd() + 1);
                    j++;
                }
            }
        }
    }
    return result;
}

Example 5 with ValueRange

use of com.microsoft.frameworklauncher.common.model.ValueRange in project pai by Microsoft.

the class ValueRangeUtilsTest method ValueRangeUtilsTest.

@Test
public void ValueRangeUtilsTest() {
    List<ValueRange> testRangeList = new ArrayList<ValueRange>();
    testRangeList.add(ValueRange.newInstance(6, 7));
    testRangeList.add(ValueRange.newInstance(10, 100));
    testRangeList.add(ValueRange.newInstance(3, 5));
    testRangeList.add(ValueRange.newInstance(90, 102));
    List<ValueRange> testRangeList2 = new ArrayList<ValueRange>();
    testRangeList2.add(ValueRange.newInstance(2, 3));
    testRangeList2.add(ValueRange.newInstance(7, 8));
    testRangeList2.add(ValueRange.newInstance(10, 20));
    List<ValueRange> testRangeList3 = ValueRangeUtils.cloneList(testRangeList2);
    List<ValueRange> sortedResult = ValueRangeUtils.SortRangeList(testRangeList);
    Assert.assertEquals(3, sortedResult.get(0).getBegin().intValue());
    Assert.assertEquals(6, sortedResult.get(1).getBegin().intValue());
    Assert.assertEquals(10, sortedResult.get(2).getBegin().intValue());
    Assert.assertEquals(90, sortedResult.get(3).getBegin().intValue());
    List<ValueRange> coalesceResult = ValueRangeUtils.coalesceRangeList(testRangeList);
    Assert.assertEquals(2, coalesceResult.size());
    Assert.assertEquals(3, coalesceResult.get(0).getBegin().intValue());
    Assert.assertEquals(7, coalesceResult.get(0).getEnd().intValue());
    Assert.assertEquals(10, coalesceResult.get(1).getBegin().intValue());
    Assert.assertEquals(102, coalesceResult.get(1).getEnd().intValue());
    List<ValueRange> result = ValueRangeUtils.intersectRangeList(coalesceResult, testRangeList2);
    Assert.assertEquals(3, result.size());
    Assert.assertEquals(3, result.get(0).getBegin().intValue());
    Assert.assertEquals(3, result.get(0).getEnd().intValue());
    Assert.assertEquals(7, result.get(1).getBegin().intValue());
    Assert.assertEquals(7, result.get(1).getEnd().intValue());
    Assert.assertEquals(10, result.get(2).getBegin().intValue());
    Assert.assertEquals(20, result.get(2).getEnd().intValue());
    result = ValueRangeUtils.subtractRange(coalesceResult, testRangeList2);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals(4, result.get(0).getBegin().intValue());
    Assert.assertEquals(6, result.get(0).getEnd().intValue());
    Assert.assertEquals(21, result.get(1).getBegin().intValue());
    Assert.assertEquals(102, result.get(1).getEnd().intValue());
    List<ValueRange> testRangeList7 = new ArrayList<ValueRange>();
    testRangeList7.add(ValueRange.newInstance(80, 80));
    result = ValueRangeUtils.subtractRange(coalesceResult, testRangeList7);
    Assert.assertEquals(3, result.size());
    Assert.assertEquals(3, result.get(0).getBegin().intValue());
    Assert.assertEquals(7, result.get(0).getEnd().intValue());
    Assert.assertEquals(10, result.get(1).getBegin().intValue());
    Assert.assertEquals(79, result.get(1).getEnd().intValue());
    Assert.assertEquals(81, result.get(2).getBegin().intValue());
    Assert.assertEquals(102, result.get(2).getEnd().intValue());
    result = ValueRangeUtils.addRange(sortedResult, testRangeList2);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals(2, result.get(0).getBegin().intValue());
    Assert.assertEquals(8, result.get(0).getEnd().intValue());
    Assert.assertEquals(10, result.get(1).getBegin().intValue());
    Assert.assertEquals(102, result.get(1).getEnd().intValue());
    List<ValueRange> testRangeList4 = new ArrayList<ValueRange>();
    testRangeList4.add(ValueRange.newInstance(2, 3));
    Assert.assertTrue(ValueRangeUtils.fitInRange(testRangeList4, testRangeList3));
    List<ValueRange> testRangeList5 = new ArrayList<ValueRange>();
    testRangeList5.add(ValueRange.newInstance(1, 3));
    Assert.assertTrue(!ValueRangeUtils.fitInRange(testRangeList5, testRangeList3));
    List<ValueRange> testRangeList6 = new ArrayList<ValueRange>();
    testRangeList6.add(ValueRange.newInstance(9, 9));
    Assert.assertTrue(!ValueRangeUtils.fitInRange(testRangeList6, testRangeList3));
    result = ValueRangeUtils.getSubRange(testRangeList3, 1, 0);
    Assert.assertEquals(1, result.size());
    Assert.assertEquals(1, ValueRangeUtils.getValueNumber(result));
    result = ValueRangeUtils.getSubRange(testRangeList3, 3, 0);
    Assert.assertEquals(3, ValueRangeUtils.getValueNumber(result));
    result = ValueRangeUtils.getSubRange(testRangeList3, 3, 10);
    Assert.assertEquals(3, ValueRangeUtils.getValueNumber(result));
    Assert.assertTrue(result.get(0).getBegin() > 10);
    Assert.assertTrue(result.get(0).getEnd() > 10);
    List<ValueRange> testRangeList10 = new ArrayList<ValueRange>();
    testRangeList10.add(ValueRange.newInstance(80, 80));
    testRangeList10.add(ValueRange.newInstance(80, 81));
    testRangeList10.add(ValueRange.newInstance(100, 103));
    int[] expectedResult = { 80, 81, 100, 101, 102, 103 };
    for (int i = 0; i < ValueRangeUtils.getValueNumber(testRangeList10); i++) {
        Assert.assertEquals(expectedResult[i], ValueRangeUtils.getValue(testRangeList10, i).intValue());
    }
    Map<String, Ports> portDefinitions = new HashMap<String, Ports>();
    Ports ports1 = new Ports();
    ports1.setCount(2);
    ports1.setStart(8000);
    portDefinitions.put("http_Port", ports1);
    Ports ports2 = new Ports();
    ports2.setCount(4);
    ports2.setStart(9000);
    portDefinitions.put("http_SSH", ports2);
    List<ValueRange> testValueRange = new ArrayList<ValueRange>();
    testValueRange.add(ValueRange.newInstance(8000, 8001));
    List<ValueRange> testValueRange2 = new ArrayList<ValueRange>();
    testValueRange.add(ValueRange.newInstance(9000, 9003));
    String environmentString = ValueRangeUtils.toEnviromentVariableString(testValueRange, portDefinitions);
    Assert.assertEquals(environmentString, "http_SSH:9000,9001,9002,9003;http_Port:8000,8001;");
    Map<String, Ports> portDefinitions2 = new HashMap<String, Ports>();
    Ports ports3 = new Ports();
    ports3.setCount(2);
    ports3.setStart(0);
    portDefinitions2.put("http_Port", ports3);
    Ports ports4 = new Ports();
    ports4.setCount(4);
    ports4.setStart(0);
    portDefinitions2.put("http_SSH", ports4);
    environmentString = ValueRangeUtils.toEnviromentVariableString(testRangeList, portDefinitions2);
    Assert.assertEquals(environmentString.split(";").length, 2);
    Assert.assertEquals(environmentString.split(",").length, 5);
}