Examples with ByteAmount com.twitter.heron.common.basics.ByteAmount used on opensource projects

Example 76 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class ResourceCompliantRRPackingTest method scaleDownAndUpWithExtraPadding.

/**
   * Test the scenario where scaling down and up is simultaneously requested and padding is
   * configured
   */
@Test
public void scaleDownAndUpWithExtraPadding() throws Exception {
    int paddingPercentage = 50;
    int numContainers = 1;
    topologyConfig.setContainerPaddingPercentage(paddingPercentage);
    // Explicit set resources for container
    ByteAmount maxContainerRam = ByteAmount.fromGigabytes(12);
    // Explicit set component ram map
    ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setContainerRamRequested(maxContainerRam);
    topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
    topologyConfig.setNumStmgrs(numContainers);
    int noBolts = 2;
    int noSpouts = 1;
    TopologyAPI.Topology topologyExplicitRamMap = getTopology(noSpouts, noBolts, topologyConfig);
    int spoutScalingUp = 1;
    int boltScalingDown = -2;
    Map<String, Integer> componentChanges = new HashMap<>();
    // 2 spouts
    componentChanges.put(SPOUT_NAME, spoutScalingUp);
    // 0 bolts
    componentChanges.put(BOLT_NAME, boltScalingDown);
    int numContainersBeforeRepack = 1;
    PackingPlan newPackingPlan = doScalingTest(topologyExplicitRamMap, componentChanges, instanceDefaultResources.getRam(), noBolts, spoutRam, noSpouts, numContainersBeforeRepack, noSpouts + noBolts);
    Assert.assertEquals(1, newPackingPlan.getContainers().size());
    Assert.assertEquals((Integer) (noSpouts + noBolts + spoutScalingUp + boltScalingDown), newPackingPlan.getInstanceCount());
    AssertPacking.assertNumInstances(newPackingPlan.getContainers(), BOLT_NAME, noBolts + boltScalingDown);
    AssertPacking.assertNumInstances(newPackingPlan.getContainers(), SPOUT_NAME, noSpouts + spoutScalingUp);
}

Example 77 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class RoundRobinPackingTest method testCheckFailure.

@Test(expected = PackingException.class)
public void testCheckFailure() throws Exception {
    int numContainers = 2;
    int spoutParallelism = 4;
    int boltParallelism = 3;
    // Set up the topology and its config
    com.twitter.heron.api.Config topologyConfig = new com.twitter.heron.api.Config();
    topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
    // Explicit set insufficient ram for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(0);
    topologyConfig.setContainerRamRequested(containerRam);
    TopologyAPI.Topology topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    getRoundRobinPackingPlan(topology);
}

Example 78 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class PackingTestUtils method testContainerPlan.

@SafeVarargs
public static PackingPlan.ContainerPlan testContainerPlan(int containerId, Pair<String, Integer>... instanceInfo) {
    double cpu = 1.5;
    ByteAmount ram = ByteAmount.fromGigabytes(1);
    Set<PackingPlan.InstancePlan> instancePlans = new HashSet<>();
    for (Pair<String, Integer> info : instanceInfo) {
        PackingPlan.InstancePlan instance = testInstancePlan(info.first, info.second);
        instancePlans.add(instance);
        cpu += instance.getResource().getCpu();
        ram = ram.plus(instance.getResource().getRam());
    }
    Resource resource = new Resource(cpu, ram, ram);
    return new PackingPlan.ContainerPlan(containerId, instancePlans, resource);
}

Example 79 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class HeronMasterDriverTest method requestContainerForWorkerSubmitsValidRequest.

@Test
public void requestContainerForWorkerSubmitsValidRequest() {
    ByteAmount memory = ByteAmount.fromMegabytes(786);
    EvaluatorRequest request = spyDriver.createEvaluatorRequest(5, memory);
    doReturn(request).when(spyDriver).createEvaluatorRequest(5, memory);
    HeronMasterDriver.HeronWorker worker = new HeronMasterDriver.HeronWorker(3, 5, memory);
    spyDriver.requestContainerForWorker(3, worker);
    verify(mockRequestor, times(1)).submit(request);
}

Example 80 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class MesosSchedulerTest method testGetBaseContainer.

@Test
public void testGetBaseContainer() throws Exception {
    final double CPU = 0.5;
    final ByteAmount MEM = ByteAmount.fromMegabytes(100);
    final ByteAmount DISK = ByteAmount.fromMegabytes(100);
    Resource containerResources = new Resource(CPU, MEM, DISK);
    PackingPlan.ContainerPlan containerPlan = new PackingPlan.ContainerPlan(0, new HashSet<PackingPlan.InstancePlan>(), containerResources);
    Set<PackingPlan.ContainerPlan> containerPlans = new HashSet<>();
    containerPlans.add(containerPlan);
    PackingPlan packingPlan = new PackingPlan(TOPOLOGY_NAME, containerPlans);
    BaseContainer container = scheduler.getBaseContainer(0, packingPlan);
    // Assert we have constructed the correct BaseContainer structure
    Assert.assertEquals(ROLE, container.runAsUser);
    Assert.assertEquals(CPU, container.cpu, 0.01);
    Assert.assertEquals(MEM, ByteAmount.fromMegabytes(((Double) container.memInMB).longValue()));
    Assert.assertEquals(DISK, ByteAmount.fromMegabytes(((Double) container.diskInMB).longValue()));
    Assert.assertEquals(SchedulerUtils.PORTS_REQUIRED_FOR_EXECUTOR, container.ports);
    Assert.assertEquals(2, container.dependencies.size());
    Assert.assertTrue(container.dependencies.contains(CORE_PACKAGE_URI));
    Assert.assertTrue(container.dependencies.contains(TOPOLOGY_PACKAGE_URI));
    Assert.assertTrue(container.environmentVariables.isEmpty());
    Assert.assertTrue(container.name.startsWith("container_0"));
    // Convert to JSON
    String str = container.toString();
    String json = BaseContainer.getJobDefinitionInJSON(container);
    Assert.assertEquals(json, str);
    // Convert the JSON back to BaseContainer
    BaseContainer newContainer = BaseContainer.getJobFromJSONString(json);
    Assert.assertEquals(json, BaseContainer.getJobDefinitionInJSON(newContainer));
}