Examples with Resource org.apache.heron.spi.packing.Resource used on opensource projects

Example 11 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class HeronMasterDriver method scheduleHeronWorkers.

/**
 * Container allocation is asynchronous. Requests all containers in the input set serially
 * to ensure allocated resources match the required resources.
 */
void scheduleHeronWorkers(Set<ContainerPlan> containers) throws ContainerAllocationException {
    for (ContainerPlan containerPlan : containers) {
        int id = containerPlan.getId();
        if (containerPlans.containsKey(containerPlan.getId())) {
            throw new ContainerAllocationException("Received duplicate allocation request for " + id);
        }
        Resource reqResource = containerPlan.getRequiredResource();
        containerPlans.put(id, containerPlan);
        requestContainerForWorker(id, new HeronWorker(id, reqResource));
    }
}

Example 12 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class MarathonScheduler method getTopologyConf.

protected String getTopologyConf(PackingPlan packing) {
    config = Config.newBuilder().putAll(config).put(Key.TOPOLOGY_BINARY_FILE, Context.topologyBinaryFile(config)).build();
    ObjectMapper mapper = new ObjectMapper();
    // TODO (nlu): use heterogeneous resources
    // Align resources to maximal requested resource
    PackingPlan updatedPackingPlan = packing.cloneWithHomogeneousScheduledResource();
    SchedulerUtils.persistUpdatedPackingPlan(Runtime.topologyName(runtime), updatedPackingPlan, Runtime.schedulerStateManagerAdaptor(runtime));
    Resource containerResource = updatedPackingPlan.getContainers().iterator().next().getRequiredResource();
    // Create app conf list for each container
    ArrayNode instances = mapper.createArrayNode();
    for (int i = 0; i < Runtime.numContainers(runtime); i++) {
        ObjectNode instance = mapper.createObjectNode();
        instance.put(MarathonConstants.ID, marathonGroupId + "/" + Integer.toString(i));
        instance.put(MarathonConstants.COMMAND, getExecutorCommand(i));
        instance.put(MarathonConstants.CPU, containerResource.getCpu());
        instance.set(MarathonConstants.CONTAINER, getContainer(mapper));
        instance.put(MarathonConstants.MEMORY, containerResource.getRam().asMegabytes());
        instance.put(MarathonConstants.DISK, containerResource.getDisk().asMegabytes());
        instance.put(MarathonConstants.INSTANCES, 1);
        instance.set(MarathonConstants.LABELS, getLabels(mapper));
        instance.set(MarathonConstants.FETCH, getFetchList(mapper));
        instances.add(instance);
    }
    // Create marathon group for a topology
    ObjectNode appConf = mapper.createObjectNode();
    appConf.put(MarathonConstants.ID, marathonGroupId);
    appConf.set(MarathonConstants.APPS, instances);
    return appConf.toString();
}

Example 13 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class JsonFormatterUtils method renderContainerPlan.

ObjectNode renderContainerPlan(PackingPlan.ContainerPlan containerPlan) {
    Resource requiredResources = containerPlan.getRequiredResource();
    ObjectNode resources = mapper.createObjectNode();
    resources.put("cpu", requiredResources.getCpu());
    resources.put("ram", requiredResources.getRam().asBytes());
    resources.put("disk", requiredResources.getDisk().asBytes());
    ObjectNode containerNode = mapper.createObjectNode();
    containerNode.put("id", containerPlan.getId());
    containerNode.set("resources", resources);
    ArrayNode components = mapper.createArrayNode();
    for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
        components.add(renderInstancePlan(instancePlan));
    }
    containerNode.set("components", components);
    return containerNode;
}

Example 14 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class RoundRobinPackingTest method testContainerRequestedResources.

/**
 * Test the scenario container level resource config are set
 */
@Test
public void testContainerRequestedResources() throws Exception {
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(10);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    double containerCpu = 30;
    Resource containerResource = new Resource(containerCpu, containerRam, containerDisk);
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlan = doPackingTestWithPartialResource(topology, Optional.empty(), Optional.empty(), boltParallelism, Optional.empty(), Optional.empty(), spoutParallelism, numContainers, getDefaultPadding(), containerResource);
    for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> differentResources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            differentResources.add(instancePlan.getResource());
        }
        Assert.assertEquals(1, differentResources.size());
        int instancesCount = containerPlan.getInstances().size();
        Assert.assertEquals(containerRam.minus(RoundRobinPacking.DEFAULT_RAM_PADDING_PER_CONTAINER).divide(instancesCount), differentResources.iterator().next().getRam());
        Assert.assertEquals((containerCpu - RoundRobinPacking.DEFAULT_CPU_PADDING_PER_CONTAINER) / instancesCount, differentResources.iterator().next().getCpu(), DELTA);
    }
}

Example 15 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class PackingUtilsTest method testResourceScaleUp.

@Test
public void testResourceScaleUp() {
    int noSpouts = 6;
    int noBolts = 3;
    int boltScalingUp = 2;
    org.apache.heron.api.Config topologyConfig = new org.apache.heron.api.Config();
    TopologyAPI.Topology topology = getTopology(noSpouts, noBolts, topologyConfig);
    Config config = PackingTestUtils.newTestConfig(topology);
    Resource defaultInstanceResources = new Resource(Context.instanceCpu(config), Context.instanceRam(config), Context.instanceDisk(config));
    Map<String, Integer> componentChanges = new HashMap<>();
    // 5 bolts
    componentChanges.put("bolt", boltScalingUp);
    Resource scaleupResource = PackingUtils.computeTotalResourceChange(topology, componentChanges, defaultInstanceResources, PackingUtils.ScalingDirection.UP);
    Assert.assertEquals((long) (boltScalingUp * defaultInstanceResources.getCpu()), (long) scaleupResource.getCpu());
    Assert.assertEquals(defaultInstanceResources.getRam().multiply(boltScalingUp), scaleupResource.getRam());
    Assert.assertEquals(defaultInstanceResources.getDisk().multiply(boltScalingUp), scaleupResource.getDisk());
}