Examples with ByteAmount com.twitter.heron.common.basics.ByteAmount used on opensource projects

Example 1 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class HeronMasterDriver method findLargestFittingWorker.

/**
   * YARN allocates resources in fixed increments of memory and cpu. As a result, the actual
   * resource allocation may be little more than what was requested. This method finds the biggest
   * heron containerPlan that will fit the allocated YARN container. In some cases the YARN CPU
   * scheduling may be disabled, resulting in default core allocation to each container. This
   * method can ignore core fitting in such a case.
   */
@VisibleForTesting
Optional<HeronWorker> findLargestFittingWorker(AllocatedEvaluator evaluator, Collection<HeronWorker> pendingWorkers, boolean ignoreCpu) {
    ByteAmount allocatedRam = ByteAmount.fromMegabytes(evaluator.getEvaluatorDescriptor().getMemory());
    int allocatedCores = evaluator.getEvaluatorDescriptor().getNumberOfCores();
    HeronWorker biggestFittingWorker = null;
    for (HeronWorker worker : pendingWorkers) {
        if (worker.mem.greaterThan(allocatedRam)) {
            continue;
        }
        if (!ignoreCpu) {
            if (worker.cores > allocatedCores) {
                continue;
            }
        }
        if (biggestFittingWorker != null) {
            if (worker.mem.lessThan(biggestFittingWorker.mem) || worker.cores < biggestFittingWorker.cores) {
                continue;
            }
        }
        biggestFittingWorker = worker;
    }
    return Optional.fromNullable(biggestFittingWorker);
}

Example 2 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class FormatterUtils method renderResourceUsage.

public static String renderResourceUsage(Resource resource) {
    double cpu = resource.getCpu();
    ByteAmount ram = resource.getRam();
    ByteAmount disk = resource.getDisk();
    return String.format("CPU: %s, RAM: %sGB, Disk: %sGB", cpu, ram.asGigabytes(), disk.asGigabytes());
}

Example 3 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class FirstFitDecreasingPacking method setPackingConfigs.

/**
   * Instatiate the packing algorithm parameters related to this topology.
   */
private void setPackingConfigs(Config config) {
    List<TopologyAPI.Config.KeyValue> topologyConfig = topology.getTopologyConfig().getKvsList();
    this.defaultInstanceResources = new Resource(Context.instanceCpu(config), Context.instanceRam(config), Context.instanceDisk(config));
    this.paddingPercentage = TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_PADDING_PERCENTAGE, DEFAULT_CONTAINER_PADDING_PERCENTAGE);
    double defaultCpu = this.defaultInstanceResources.getCpu() * DEFAULT_NUMBER_INSTANCES_PER_CONTAINER;
    ByteAmount defaultRam = this.defaultInstanceResources.getRam().multiply(DEFAULT_NUMBER_INSTANCES_PER_CONTAINER);
    ByteAmount defaultDisk = this.defaultInstanceResources.getDisk().multiply(DEFAULT_NUMBER_INSTANCES_PER_CONTAINER);
    this.maxContainerResources = new Resource(TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_MAX_CPU_HINT, (double) Math.round(PackingUtils.increaseBy(defaultCpu, paddingPercentage))), TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_MAX_RAM_HINT, defaultRam.increaseBy(paddingPercentage)), TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_MAX_DISK_HINT, defaultDisk.increaseBy(paddingPercentage)));
}

Example 4 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class Container method getTotalUsedResources.

/**
   * Computes the used resources of the container by taking into account the resources
   * allocated for each instance.
   *
   * @return a Resource object that describes the used cpu, ram and disk in the container.
   */
private Resource getTotalUsedResources() {
    ByteAmount usedRam = ByteAmount.ZERO;
    double usedCpuCores = 0;
    ByteAmount usedDisk = ByteAmount.ZERO;
    for (PackingPlan.InstancePlan instancePlan : this.instances) {
        Resource resource = instancePlan.getResource();
        usedRam = usedRam.plus(resource.getRam());
        usedCpuCores += resource.getCpu();
        usedDisk = usedDisk.plus(resource.getDisk());
    }
    return new Resource(usedCpuCores, usedRam, usedDisk);
}

Example 5 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class Container method assertHasSpace.

/**
   * Check whether the container can accommodate a new instance with specific resource requirements
   */
private void assertHasSpace(Resource resource) throws ResourceExceededException {
    Resource usedResources = this.getTotalUsedResources();
    ByteAmount newRam = usedResources.getRam().plus(resource.getRam()).increaseBy(paddingPercentage);
    double newCpu = Math.round(PackingUtils.increaseBy(usedResources.getCpu() + resource.getCpu(), paddingPercentage));
    ByteAmount newDisk = usedResources.getDisk().plus(resource.getDisk()).increaseBy(paddingPercentage);
    if (newRam.greaterThan(this.capacity.getRam())) {
        throw new ResourceExceededException(String.format("Adding %s bytes of ram to existing %s " + "bytes with %d percent padding would exceed capacity %s", resource.getRam(), usedResources.getRam(), paddingPercentage, this.capacity.getRam()));
    }
    if (newCpu > this.capacity.getCpu()) {
        throw new ResourceExceededException(String.format("Adding %s cores to existing %s " + "cores with %d percent padding would exceed capacity %s", resource.getCpu(), usedResources.getCpu(), paddingPercentage, this.capacity.getCpu()));
    }
    if (newDisk.greaterThan(this.capacity.getDisk())) {
        throw new ResourceExceededException(String.format("Adding %s bytes of disk to existing %s " + "bytes with %s percent padding would exceed capacity %s", resource.getDisk(), usedResources.getDisk(), paddingPercentage, this.capacity.getDisk()));
    }
}