Examples with Resource com.twitter.heron.spi.packing.Resource used on opensource projects

Example 1 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class AuroraScheduler method onSchedule.

@Override
public boolean onSchedule(PackingPlan packing) {
    if (packing == null || packing.getContainers().isEmpty()) {
        LOG.severe("No container requested. Can't schedule");
        return false;
    }
    LOG.info("Launching topology in aurora");
    // Align the cpu, ram, disk to the maximal one
    PackingPlan updatedPackingPlan = packing.cloneWithHomogeneousScheduledResource();
    SchedulerUtils.persistUpdatedPackingPlan(Runtime.topologyName(runtime), updatedPackingPlan, Runtime.schedulerStateManagerAdaptor(runtime));
    Resource containerResource = packing.getContainers().iterator().next().getRequiredResource();
    Map<AuroraField, String> auroraProperties = createAuroraProperties(containerResource);
    return controller.createJob(auroraProperties);
}

Example 2 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class FirstFitDecreasingPacking method setPackingConfigs.

/**
   * Instatiate the packing algorithm parameters related to this topology.
   */
private void setPackingConfigs(Config config) {
    List<TopologyAPI.Config.KeyValue> topologyConfig = topology.getTopologyConfig().getKvsList();
    this.defaultInstanceResources = new Resource(Context.instanceCpu(config), Context.instanceRam(config), Context.instanceDisk(config));
    this.paddingPercentage = TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_PADDING_PERCENTAGE, DEFAULT_CONTAINER_PADDING_PERCENTAGE);
    double defaultCpu = this.defaultInstanceResources.getCpu() * DEFAULT_NUMBER_INSTANCES_PER_CONTAINER;
    ByteAmount defaultRam = this.defaultInstanceResources.getRam().multiply(DEFAULT_NUMBER_INSTANCES_PER_CONTAINER);
    ByteAmount defaultDisk = this.defaultInstanceResources.getDisk().multiply(DEFAULT_NUMBER_INSTANCES_PER_CONTAINER);
    this.maxContainerResources = new Resource(TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_MAX_CPU_HINT, (double) Math.round(PackingUtils.increaseBy(defaultCpu, paddingPercentage))), TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_MAX_RAM_HINT, defaultRam.increaseBy(paddingPercentage)), TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_MAX_DISK_HINT, defaultDisk.increaseBy(paddingPercentage)));
}

Example 3 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class Container method getTotalUsedResources.

/**
   * Computes the used resources of the container by taking into account the resources
   * allocated for each instance.
   *
   * @return a Resource object that describes the used cpu, ram and disk in the container.
   */
private Resource getTotalUsedResources() {
    ByteAmount usedRam = ByteAmount.ZERO;
    double usedCpuCores = 0;
    ByteAmount usedDisk = ByteAmount.ZERO;
    for (PackingPlan.InstancePlan instancePlan : this.instances) {
        Resource resource = instancePlan.getResource();
        usedRam = usedRam.plus(resource.getRam());
        usedCpuCores += resource.getCpu();
        usedDisk = usedDisk.plus(resource.getDisk());
    }
    return new Resource(usedCpuCores, usedRam, usedDisk);
}

Example 4 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class Container method assertHasSpace.

/**
   * Check whether the container can accommodate a new instance with specific resource requirements
   */
private void assertHasSpace(Resource resource) throws ResourceExceededException {
    Resource usedResources = this.getTotalUsedResources();
    ByteAmount newRam = usedResources.getRam().plus(resource.getRam()).increaseBy(paddingPercentage);
    double newCpu = Math.round(PackingUtils.increaseBy(usedResources.getCpu() + resource.getCpu(), paddingPercentage));
    ByteAmount newDisk = usedResources.getDisk().plus(resource.getDisk()).increaseBy(paddingPercentage);
    if (newRam.greaterThan(this.capacity.getRam())) {
        throw new ResourceExceededException(String.format("Adding %s bytes of ram to existing %s " + "bytes with %d percent padding would exceed capacity %s", resource.getRam(), usedResources.getRam(), paddingPercentage, this.capacity.getRam()));
    }
    if (newCpu > this.capacity.getCpu()) {
        throw new ResourceExceededException(String.format("Adding %s cores to existing %s " + "cores with %d percent padding would exceed capacity %s", resource.getCpu(), usedResources.getCpu(), paddingPercentage, this.capacity.getCpu()));
    }
    if (newDisk.greaterThan(this.capacity.getDisk())) {
        throw new ResourceExceededException(String.format("Adding %s bytes of disk to existing %s " + "bytes with %s percent padding would exceed capacity %s", resource.getDisk(), usedResources.getDisk(), paddingPercentage, this.capacity.getDisk()));
    }
}

Example 5 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class PackingPlanBuilder method getContainers.

/**
   * Generates the containers that correspond to the current packing plan
   * along with their associated instances.
   *
   * @return Map of containers for the current packing plan, keyed by containerId
   */
@VisibleForTesting
static Map<Integer, Container> getContainers(PackingPlan currentPackingPlan, int paddingPercentage, Map<String, TreeSet<Integer>> componentIndexes, TreeSet<Integer> taskIds) throws ResourceExceededException {
    Map<Integer, Container> containers = new HashMap<>();
    Resource capacity = currentPackingPlan.getMaxContainerResources();
    for (PackingPlan.ContainerPlan currentContainerPlan : currentPackingPlan.getContainers()) {
        Container container = new Container(currentContainerPlan.getId(), capacity, paddingPercentage);
        for (PackingPlan.InstancePlan instancePlan : currentContainerPlan.getInstances()) {
            try {
                addToContainer(container, instancePlan, componentIndexes, taskIds);
            } catch (ResourceExceededException e) {
                throw new ResourceExceededException(String.format("Insufficient container resources to add instancePlan %s to container %s", instancePlan, container), e);
            }
        }
        containers.put(currentContainerPlan.getId(), container);
    }
    return containers;
}