Examples with Resource com.twitter.heron.spi.packing.Resource used on opensource projects

Example 16 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class MesosScheduler method fillResourcesRequirementForBaseContainer.

/**
   * Fill the the resources requirement, i.e. cpu, memory and disk for the given container.
   * This method changes the BaseContainer passed in.
   * <p>
   * Notice: Currently we just make every container homogeneous,
   * requiring maximum resources for every container.
   *
   * @param container the BaseContainer to fill value in
   * @param containerIndex the index of the container
   * @param packing the packing plan
   */
protected void fillResourcesRequirementForBaseContainer(BaseContainer container, Integer containerIndex, PackingPlan packing) {
    PackingPlan updatedPackingPlan = packing.cloneWithHomogeneousScheduledResource();
    Resource maxResourceContainer = updatedPackingPlan.getContainers().iterator().next().getRequiredResource();
    double cpu = 0;
    ByteAmount disk = ByteAmount.ZERO;
    ByteAmount mem = ByteAmount.ZERO;
    for (PackingPlan.ContainerPlan cp : packing.getContainers()) {
        Resource containerResource = cp.getRequiredResource();
        cpu = Math.max(cpu, containerResource.getCpu());
        disk = disk.max(containerResource.getDisk());
        mem = mem.max(containerResource.getRam());
    }
    container.cpu = maxResourceContainer.getCpu();
    // Convert them from bytes to MB
    container.diskInMB = maxResourceContainer.getDisk().asMegabytes();
    container.memInMB = maxResourceContainer.getRam().asMegabytes();
    container.ports = SchedulerUtils.PORTS_REQUIRED_FOR_EXECUTOR;
}

Example 17 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class PackingPlanBuilder method addInstance.

// adds an instance to a container with id containerId. If that container does not exist, it will
// be lazily initialized, which could result in more containers than those requested using the
// updateNumContainers method
public PackingPlanBuilder addInstance(Integer containerId, String componentName) throws ResourceExceededException {
    initContainer(containerId);
    Integer taskId = taskIds.isEmpty() ? 1 : taskIds.last() + 1;
    Integer componentIndex = componentIndexes.get(componentName) != null ? componentIndexes.get(componentName).last() + 1 : 0;
    InstanceId instanceId = new InstanceId(componentName, taskId, componentIndex);
    Resource instanceResource = PackingUtils.getResourceRequirement(componentName, this.componentRamMap, this.defaultInstanceResource, this.maxContainerResource, this.requestedContainerPadding);
    try {
        addToContainer(containers.get(containerId), new PackingPlan.InstancePlan(instanceId, instanceResource), this.componentIndexes, this.taskIds);
    } catch (ResourceExceededException e) {
        throw new ResourceExceededException(String.format("Insufficient container resources to add instance %s with resources %s to container %d.", instanceId, instanceResource, containerId), e);
    }
    LOG.finest(String.format("Added to container %d instance %s", containerId, instanceId));
    return this;
}

Example 18 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class PackingPlanBuilder method initContainers.

private void initContainers() {
    assertResourceSettings();
    Map<Integer, Container> newContainerMap = this.containers;
    HashMap<String, TreeSet<Integer>> newComponentIndexes = this.componentIndexes;
    TreeSet<Integer> newTaskIds = this.taskIds;
    if (newComponentIndexes == null) {
        newComponentIndexes = new HashMap<>();
    }
    if (newTaskIds == null) {
        newTaskIds = new TreeSet<>();
    }
    // if this is the first time called, initialize container map with empty or existing containers
    if (newContainerMap == null) {
        if (this.existingPacking == null) {
            newContainerMap = new HashMap<>();
            for (int containerId = 1; containerId <= numContainers; containerId++) {
                newContainerMap.put(containerId, new Container(containerId, this.maxContainerResource, this.requestedContainerPadding));
            }
        } else {
            try {
                newContainerMap = getContainers(this.existingPacking, this.requestedContainerPadding, newComponentIndexes, newTaskIds);
            } catch (ResourceExceededException e) {
                throw new PackingException("Could not initialize containers using existing packing plan", e);
            }
        }
    }
    if (this.numContainers > newContainerMap.size()) {
        List<Scorer<Container>> scorers = new ArrayList<>();
        scorers.add(new ContainerIdScorer());
        List<Container> sortedContainers = sortContainers(scorers, newContainerMap.values());
        int nextContainerId = sortedContainers.get(sortedContainers.size() - 1).getContainerId() + 1;
        Resource capacity = newContainerMap.get(sortedContainers.get(0).getContainerId()).getCapacity();
        for (int i = 0; i < numContainers - newContainerMap.size(); i++) {
            newContainerMap.put(nextContainerId, new Container(nextContainerId, capacity, this.requestedContainerPadding));
            nextContainerId++;
        }
    }
    this.containers = newContainerMap;
    this.componentIndexes = newComponentIndexes;
    this.taskIds = newTaskIds;
}

Example 19 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class ResourceCompliantRRPacking method newPackingPlanBuilder.

private PackingPlanBuilder newPackingPlanBuilder(PackingPlan existingPackingPlan) {
    List<TopologyAPI.Config.KeyValue> topologyConfig = topology.getTopologyConfig().getKvsList();
    double defaultCpu = this.defaultInstanceResources.getCpu() * DEFAULT_NUMBER_INSTANCES_PER_CONTAINER;
    ByteAmount defaultRam = this.defaultInstanceResources.getRam().multiply(DEFAULT_NUMBER_INSTANCES_PER_CONTAINER);
    ByteAmount defaultDisk = this.defaultInstanceResources.getDisk().multiply(DEFAULT_NUMBER_INSTANCES_PER_CONTAINER);
    int paddingPercentage = TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_PADDING_PERCENTAGE, DEFAULT_CONTAINER_PADDING_PERCENTAGE);
    Resource maxContainerResources = new Resource(TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_CPU_REQUESTED, (double) Math.round(PackingUtils.increaseBy(defaultCpu, paddingPercentage))), TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_RAM_REQUESTED, defaultRam.increaseBy(paddingPercentage)), TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_DISK_REQUESTED, defaultDisk.increaseBy(paddingPercentage)));
    return new PackingPlanBuilder(topology.getId(), existingPackingPlan).setMaxContainerResource(maxContainerResources).setDefaultInstanceResource(defaultInstanceResources).setRequestedContainerPadding(paddingPercentage).setRequestedComponentRam(TopologyUtils.getComponentRamMapConfig(topology));
}

Example 20 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class ResourceCompliantRRPacking method computeNumAdditionalContainers.

/**
   * Computes the additional number of containers needed to accommodate a scale up/down operation
   *
   * @param componentChanges parallelism changes for scale up/down
   * @param packingPlan existing packing plan
   * @return additional number of containers needed
   */
private int computeNumAdditionalContainers(Map<String, Integer> componentChanges, PackingPlan packingPlan) {
    Resource scaleDownResource = PackingUtils.computeTotalResourceChange(topology, componentChanges, defaultInstanceResources, PackingUtils.ScalingDirection.DOWN);
    Resource scaleUpResource = PackingUtils.computeTotalResourceChange(topology, componentChanges, defaultInstanceResources, PackingUtils.ScalingDirection.UP);
    Resource additionalResource = scaleUpResource.subtractAbsolute(scaleDownResource);
    return (int) additionalResource.divideBy(packingPlan.getMaxContainerResources());
}