Examples with ByteAmount com.twitter.heron.common.basics.ByteAmount used on opensource projects

Example 91 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class MesosSchedulerTest method testGetBaseContainer.

@Test
public void testGetBaseContainer() throws Exception {
    final double CPU = 0.5;
    final ByteAmount MEM = ByteAmount.fromMegabytes(100);
    final ByteAmount DISK = ByteAmount.fromMegabytes(100);
    Resource containerResources = new Resource(CPU, MEM, DISK);
    PackingPlan.ContainerPlan containerPlan = new PackingPlan.ContainerPlan(0, new HashSet<PackingPlan.InstancePlan>(), containerResources);
    Set<PackingPlan.ContainerPlan> containerPlans = new HashSet<>();
    containerPlans.add(containerPlan);
    PackingPlan packingPlan = new PackingPlan(TOPOLOGY_NAME, containerPlans);
    BaseContainer container = scheduler.getBaseContainer(0, packingPlan);
    // Assert we have constructed the correct BaseContainer structure
    Assert.assertEquals(ROLE, container.runAsUser);
    Assert.assertEquals(CPU, container.cpu, 0.01);
    Assert.assertEquals(MEM, ByteAmount.fromMegabytes(((Double) container.memInMB).longValue()));
    Assert.assertEquals(DISK, ByteAmount.fromMegabytes(((Double) container.diskInMB).longValue()));
    Assert.assertEquals(SchedulerUtils.ExecutorPort.getRequiredPorts().size(), container.ports);
    Assert.assertEquals(2, container.dependencies.size());
    Assert.assertTrue(container.dependencies.contains(CORE_PACKAGE_URI));
    Assert.assertTrue(container.dependencies.contains(TOPOLOGY_PACKAGE_URI));
    Assert.assertTrue(container.environmentVariables.isEmpty());
    Assert.assertTrue(container.name.startsWith("container_0"));
    // Convert to JSON
    String str = container.toString();
    String json = BaseContainer.getJobDefinitionInJSON(container);
    Assert.assertEquals(json, str);
    // Convert the JSON back to BaseContainer
    BaseContainer newContainer = BaseContainer.getJobFromJSONString(json);
    Assert.assertEquals(json, BaseContainer.getJobDefinitionInJSON(newContainer));
}

Example 92 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class HeronMasterDriverTest method requestContainerForWorkerSubmitsValidRequest.

@Test
public void requestContainerForWorkerSubmitsValidRequest() {
    ByteAmount memory = ByteAmount.fromMegabytes(786);
    EvaluatorRequest request = spyDriver.createEvaluatorRequest(5, memory);
    doReturn(request).when(spyDriver).createEvaluatorRequest(5, memory);
    HeronMasterDriver.HeronWorker worker = new HeronMasterDriver.HeronWorker(3, 5, memory);
    spyDriver.requestContainerForWorker(3, worker);
    verify(mockRequestor, times(1)).submit(request);
}

Example 93 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class ConfigBuilder method setComponentLevelResource.

private void setComponentLevelResource(Config config, Map.Entry<String, Object> entry) {
    List<Object> objects = (List<Object>) entry.getValue();
    for (Object obj : objects) {
        String objString = obj.toString();
        objString = objString.replace(COMMA, WHITESPACE);
        objString = objString.replace(LEFT_BRACE, WHITESPACE);
        objString = objString.replace(RIGHT_BRACE, WHITESPACE);
        int idIndex = objString.indexOf(ID);
        int ramIndex = objString.indexOf(RAM);
        int cpuIndex = objString.indexOf(CPU);
        int diskIndex = objString.indexOf(DISK);
        String ramWithUom = "";
        String id = getIdValue(objString, idIndex);
        // todo (josh fischer) diskWithUom and cpu are still to be implemented for use with k8s
        String diskWithUom = "";
        String cpu = "";
        if (ramIndex != -1) {
            ramWithUom = assignValue(objString, ramIndex);
        }
        if (cpuIndex != -1) {
            cpu = assignValue(objString, cpuIndex);
        }
        if (diskIndex != -1) {
            diskWithUom = assignValue(objString, diskIndex);
        }
        ByteAmount byteAmount = null;
        if (ramWithUom.contains(MB)) {
            // its megaBytes
            int mbIndex = verifyStartingIndexOfUom(ramWithUom, MB);
            long megaBytes = extractRawValue(ramWithUom, mbIndex);
            if (megaBytes < MINIMUM_MB) {
                throw new IllegalArgumentException("The minimum Ram resource allocation for a component must be at least 256MB");
            }
            byteAmount = ByteAmount.fromMegabytes(megaBytes);
        } else if (ramWithUom.contains(GB)) {
            // its gigaBytes
            // we don't validate here as NumberFormatException is thrown converting decimals to longs
            int gbIndex = verifyStartingIndexOfUom(ramWithUom, GB);
            byteAmount = ByteAmount.fromGigabytes(extractRawValue(ramWithUom, gbIndex));
        } else if (ramWithUom.contains(B)) {
            // its bytes
            int bIndex = verifyStartingIndexOfUom(ramWithUom, B);
            long bytes = extractRawValue(ramWithUom, bIndex);
            if (bytes < MINIMUM_BYTES) {
                throw new IllegalArgumentException("The minimum Ram resource allocation for a component must be at least 256000000B");
            }
            byteAmount = ByteAmount.fromBytes(bytes);
        } else {
            // There is no format throw an exception
            throw new IllegalArgumentException(" Please specify 'B', 'MB', 'GB' when declaring Ram Resources");
        }
        config.setComponentRam(id, byteAmount);
    }
}

Example 94 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class PackingUtils method getResourceRequirement.

public static Resource getResourceRequirement(String component, Map<String, ByteAmount> componentRamMap, Resource defaultInstanceResource, Resource maxContainerResource, int paddingPercentage) {
    ByteAmount instanceRam = defaultInstanceResource.getRam();
    if (componentRamMap.containsKey(component)) {
        instanceRam = componentRamMap.get(component);
    }
    assertIsValidInstance(defaultInstanceResource.cloneWithRam(instanceRam), MIN_RAM_PER_INSTANCE, maxContainerResource, paddingPercentage);
    return defaultInstanceResource.cloneWithRam(instanceRam);
}

Example 95 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class PackingUtils method computeTotalResourceChange.

/**
 * Identifies the resources reclaimed by the components that will be scaled down
 *
 * @return Total resources reclaimed
 */
public static Resource computeTotalResourceChange(TopologyAPI.Topology topology, Map<String, Integer> componentChanges, Resource defaultInstanceResources, ScalingDirection scalingDirection) {
    double cpu = 0;
    ByteAmount ram = ByteAmount.ZERO;
    ByteAmount disk = ByteAmount.ZERO;
    Map<String, ByteAmount> ramMap = TopologyUtils.getComponentRamMapConfig(topology);
    Map<String, Integer> componentsToScale = PackingUtils.getComponentsToScale(componentChanges, scalingDirection);
    for (String component : componentsToScale.keySet()) {
        int parallelismChange = Math.abs(componentChanges.get(component));
        cpu += parallelismChange * defaultInstanceResources.getCpu();
        disk = disk.plus(defaultInstanceResources.getDisk().multiply(parallelismChange));
        if (ramMap.containsKey(component)) {
            ram = ram.plus(ramMap.get(component).multiply(parallelismChange));
        } else {
            ram = ram.plus(defaultInstanceResources.getRam().multiply(parallelismChange));
        }
    }
    return new Resource(cpu, ram, disk);
}