Examples with ByteAmount com.twitter.heron.common.basics.ByteAmount used on opensource projects

Example 61 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class PackingUtils method assertIsValidInstance.

/**
   * Verifies the Instance has enough RAM and that it can fit within the container limits.
   *
   * @param instanceResources The resources allocated to the instance
   * @throws PackingException if the instance is invalid
   */
private static void assertIsValidInstance(Resource instanceResources, ByteAmount minInstanceRam, Resource maxContainerResources, int paddingPercentage) throws PackingException {
    if (instanceResources.getRam().lessThan(minInstanceRam)) {
        throw new PackingException(String.format("Instance requires ram %s which is less than the minimum ram per instance of %s", instanceResources.getRam(), minInstanceRam));
    }
    ByteAmount instanceRam = instanceResources.getRam().increaseBy(paddingPercentage);
    if (instanceRam.greaterThan(maxContainerResources.getRam())) {
        throw new PackingException(String.format("This instance requires containers of at least %s ram. The current max container " + "size is %s", instanceRam, maxContainerResources.getRam()));
    }
    double instanceCpu = Math.round(PackingUtils.increaseBy(instanceResources.getCpu(), paddingPercentage));
    if (instanceCpu > maxContainerResources.getCpu()) {
        throw new PackingException(String.format("This instance requires containers with at least %s cpu cores. The current max container" + "size is %s cores", instanceCpu > maxContainerResources.getCpu(), maxContainerResources.getCpu()));
    }
    ByteAmount instanceDisk = instanceResources.getDisk().increaseBy(paddingPercentage);
    if (instanceDisk.greaterThan(maxContainerResources.getDisk())) {
        throw new PackingException(String.format("This instance requires containers of at least %s disk. The current max container" + "size is %s", instanceDisk, maxContainerResources.getDisk()));
    }
}

Example 62 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class FirstFitDecreasingPackingTest method testContainerRequestedResources.

/**
   * Test the scenario where container level resource config are set
   */
@Test
public void testContainerRequestedResources() throws Exception {
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(10);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    float containerCpu = 30;
    topologyConfig.setContainerMaxRamHint(containerRam);
    topologyConfig.setContainerMaxDiskHint(containerDisk);
    topologyConfig.setContainerMaxCpuHint(containerCpu);
    TopologyAPI.Topology topologyExplicitResourcesConfig = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanExplicitResourcesConfig = pack(topologyExplicitResourcesConfig);
    Assert.assertEquals(1, packingPlanExplicitResourcesConfig.getContainers().size());
    Assert.assertEquals(totalInstances, packingPlanExplicitResourcesConfig.getInstanceCount());
    AssertPacking.assertNumInstances(packingPlanExplicitResourcesConfig.getContainers(), BOLT_NAME, 3);
    AssertPacking.assertNumInstances(packingPlanExplicitResourcesConfig.getContainers(), SPOUT_NAME, 4);
    for (PackingPlan.ContainerPlan containerPlan : packingPlanExplicitResourcesConfig.getContainers()) {
        Assert.assertEquals(Math.round(PackingUtils.increaseBy(totalInstances * instanceDefaultResources.getCpu(), DEFAULT_CONTAINER_PADDING)), (long) containerPlan.getRequiredResource().getCpu());
        Assert.assertEquals(instanceDefaultResources.getRam().multiply(totalInstances).increaseBy(DEFAULT_CONTAINER_PADDING), containerPlan.getRequiredResource().getRam());
        Assert.assertEquals(instanceDefaultResources.getDisk().multiply(totalInstances).increaseBy(DEFAULT_CONTAINER_PADDING), containerPlan.getRequiredResource().getDisk());
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> resources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            resources.add(instancePlan.getResource());
        }
        Assert.assertEquals(1, resources.size());
        Assert.assertEquals(instanceDefaultResources.getRam(), resources.iterator().next().getRam());
    }
}

Example 63 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class FirstFitDecreasingPackingTest method testPartialRamMap.

/**
   * Test the scenario ram map config is partially set
   */
@Test
public void testPartialRamMap() throws Exception {
    // Explicit set resources for container
    ByteAmount maxContainerRam = ByteAmount.fromGigabytes(10);
    // Explicit set component ram map
    ByteAmount boltRam = ByteAmount.fromGigabytes(4);
    topologyConfig.setContainerMaxRamHint(maxContainerRam);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    TopologyAPI.Topology topologyExplicitRamMap = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanExplicitRamMap = pack(topologyExplicitRamMap);
    Assert.assertEquals(2, packingPlanExplicitRamMap.getContainers().size());
    Assert.assertEquals(totalInstances, packingPlanExplicitRamMap.getInstanceCount());
    AssertPacking.assertNumInstances(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, 3);
    AssertPacking.assertNumInstances(packingPlanExplicitRamMap.getContainers(), SPOUT_NAME, 4);
    AssertPacking.assertContainers(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, SPOUT_NAME, boltRam, instanceDefaultResources.getRam(), maxContainerRam);
    AssertPacking.assertContainerRam(packingPlanExplicitRamMap.getContainers(), maxContainerRam);
}

Example 64 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class FirstFitDecreasingPackingTest method testDefaultContainerSize.

/**
   * Test the scenario where the max container size is the default
   */
@Test
public void testDefaultContainerSize() throws Exception {
    int defaultNumInstancesperContainer = 4;
    PackingPlan packingPlan = pack(topology);
    Assert.assertEquals(2, packingPlan.getContainers().size());
    Assert.assertEquals(totalInstances, packingPlan.getInstanceCount());
    ByteAmount defaultRam = instanceDefaultResources.getRam().multiply(defaultNumInstancesperContainer).increaseBy(DEFAULT_CONTAINER_PADDING);
    AssertPacking.assertContainerRam(packingPlan.getContainers(), defaultRam);
    AssertPacking.assertNumInstances(packingPlan.getContainers(), BOLT_NAME, 3);
    AssertPacking.assertNumInstances(packingPlan.getContainers(), SPOUT_NAME, 4);
}

Example 65 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class FirstFitDecreasingPackingTest method testCompleteRamMapRequested.

/**
   * Test the scenario ram map config is fully set
   */
@Test
public void testCompleteRamMapRequested() throws Exception {
    // Explicit set max resources for container
    // the value should be ignored, since we set the complete component ram map
    ByteAmount maxContainerRam = ByteAmount.fromGigabytes(15);
    ByteAmount maxContainerDisk = ByteAmount.fromGigabytes(20);
    float maxContainerCpu = 30;
    // Explicit set component ram map
    ByteAmount boltRam = ByteAmount.fromGigabytes(1);
    ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setContainerMaxRamHint(maxContainerRam);
    topologyConfig.setContainerMaxDiskHint(maxContainerDisk);
    topologyConfig.setContainerMaxCpuHint(maxContainerCpu);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
    TopologyAPI.Topology topologyExplicitRamMap = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanExplicitRamMap = pack(topologyExplicitRamMap);
    Assert.assertEquals(1, packingPlanExplicitRamMap.getContainers().size());
    Assert.assertEquals(totalInstances, packingPlanExplicitRamMap.getInstanceCount());
    AssertPacking.assertNumInstances(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, 3);
    AssertPacking.assertNumInstances(packingPlanExplicitRamMap.getContainers(), SPOUT_NAME, 4);
    AssertPacking.assertContainers(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, SPOUT_NAME, boltRam, spoutRam, maxContainerRam);
    AssertPacking.assertContainerRam(packingPlanExplicitRamMap.getContainers(), maxContainerRam);
}