Examples with ByteAmount com.twitter.heron.common.basics.ByteAmount used on opensource projects

Example 56 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class JVMMetrics method updateBufferPoolMetrics.

// Gather metrics related to both direct and mapped byte buffers in the jvm.
// These metrics can be useful for diagnosing native memory usage.
private void updateBufferPoolMetrics() {
    for (BufferPoolMXBean bufferPoolMXBean : bufferPoolMXBeanList) {
        String normalizedKeyName = bufferPoolMXBean.getName().replaceAll("[^\\w]", "-");
        final ByteAmount memoryUsed = ByteAmount.fromBytes(bufferPoolMXBean.getMemoryUsed());
        final ByteAmount totalCapacity = ByteAmount.fromBytes(bufferPoolMXBean.getTotalCapacity());
        final ByteAmount count = ByteAmount.fromBytes(bufferPoolMXBean.getCount());
        // The estimated memory the JVM is using for this buffer pool
        jvmBufferPoolMemoryUsage.safeScope(normalizedKeyName + "-memory-used").setValue(memoryUsed.asMegabytes());
        // The estimated total capacity of the buffers in this pool
        jvmBufferPoolMemoryUsage.safeScope(normalizedKeyName + "-total-capacity").setValue(totalCapacity.asMegabytes());
        // THe estimated number of buffers in this pool
        jvmBufferPoolMemoryUsage.safeScope(normalizedKeyName + "-count").setValue(count.asMegabytes());
    }
}

Example 57 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class JVMMetrics method getJVMSampleRunnable.

public Runnable getJVMSampleRunnable() {
    final Runnable sampleRunnable = new Runnable() {

        @Override
        public void run() {
            updateGcMetrics();
            jvmUpTimeSecs.setValue(Duration.ofMillis(runtimeMXBean.getUptime()).getSeconds());
            processCPUTimeNs.setValue(getProcessCPUTimeNs());
            getThreadsMetrics();
            // We multiple # of processors to measure a process cpu load based on cores rather than
            // overall machine
            processCPULoad.update(getProcessCPULoad() * runtime.availableProcessors());
            updateFdMetrics();
            updateMemoryPoolMetrics();
            updateBufferPoolMetrics();
            ByteAmount freeMemory = ByteAmount.fromBytes(runtime.freeMemory());
            ByteAmount totalMemory = ByteAmount.fromBytes(runtime.totalMemory());
            jvmMemoryFreeMB.update(freeMemory.asMegabytes());
            jvmMemoryTotalMB.update(totalMemory.asMegabytes());
            jvmMemoryUsedMB.update(totalMemory.asMegabytes() - freeMemory.asMegabytes());
            jvmMemoryHeapUsedMB.update(ByteAmount.fromBytes(memoryBean.getHeapMemoryUsage().getUsed()).asMegabytes());
            jvmMemoryHeapCommittedMB.update(ByteAmount.fromBytes(memoryBean.getHeapMemoryUsage().getCommitted()).asMegabytes());
            jvmMemoryHeapMaxMB.update(ByteAmount.fromBytes(memoryBean.getHeapMemoryUsage().getMax()).asMegabytes());
            jvmMemoryNonHeapUsedMB.update(ByteAmount.fromBytes(memoryBean.getNonHeapMemoryUsage().getUsed()).asMegabytes());
            jvmMemoryNonHeapCommittedMB.update(ByteAmount.fromBytes(memoryBean.getNonHeapMemoryUsage().getCommitted()).asMegabytes());
            jvmMemoryNonHeapMaxMB.update(ByteAmount.fromBytes(memoryBean.getNonHeapMemoryUsage().getMax()).asMegabytes());
        }
    };
    return sampleRunnable;
}

Example 58 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class ResourceCompliantRRPacking method newPackingPlanBuilder.

private PackingPlanBuilder newPackingPlanBuilder(PackingPlan existingPackingPlan) {
    List<TopologyAPI.Config.KeyValue> topologyConfig = topology.getTopologyConfig().getKvsList();
    double defaultCpu = this.defaultInstanceResources.getCpu() * DEFAULT_NUMBER_INSTANCES_PER_CONTAINER;
    ByteAmount defaultRam = this.defaultInstanceResources.getRam().multiply(DEFAULT_NUMBER_INSTANCES_PER_CONTAINER);
    ByteAmount defaultDisk = this.defaultInstanceResources.getDisk().multiply(DEFAULT_NUMBER_INSTANCES_PER_CONTAINER);
    int paddingPercentage = TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_PADDING_PERCENTAGE, DEFAULT_CONTAINER_PADDING_PERCENTAGE);
    Resource maxContainerResources = new Resource(TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_CPU_REQUESTED, (double) Math.round(PackingUtils.increaseBy(defaultCpu, paddingPercentage))), TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_RAM_REQUESTED, defaultRam.increaseBy(paddingPercentage)), TopologyUtils.getConfigWithDefault(topologyConfig, TOPOLOGY_CONTAINER_DISK_REQUESTED, defaultDisk.increaseBy(paddingPercentage)));
    return new PackingPlanBuilder(topology.getId(), existingPackingPlan).setMaxContainerResource(maxContainerResources).setDefaultInstanceResource(defaultInstanceResources).setRequestedContainerPadding(paddingPercentage).setRequestedComponentRam(TopologyUtils.getComponentRamMapConfig(topology));
}

Example 59 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class RoundRobinPacking method pack.

@Override
public PackingPlan pack() {
    // Get the instances' round-robin allocation
    Map<Integer, List<InstanceId>> roundRobinAllocation = getRoundRobinAllocation();
    // Get the ram map for every instance
    Map<Integer, Map<InstanceId, ByteAmount>> instancesRamMap = getInstancesRamMapInContainer(roundRobinAllocation);
    ByteAmount containerDiskInBytes = getContainerDiskHint(roundRobinAllocation);
    double containerCpu = getContainerCpuHint(roundRobinAllocation);
    // Construct the PackingPlan
    Set<PackingPlan.ContainerPlan> containerPlans = new HashSet<>();
    for (int containerId : roundRobinAllocation.keySet()) {
        List<InstanceId> instanceList = roundRobinAllocation.get(containerId);
        // Calculate the resource required for single instance
        Map<InstanceId, PackingPlan.InstancePlan> instancePlanMap = new HashMap<>();
        ByteAmount containerRam = DEFAULT_RAM_PADDING_PER_CONTAINER;
        for (InstanceId instanceId : instanceList) {
            ByteAmount instanceRam = instancesRamMap.get(containerId).get(instanceId);
            // Currently not yet support disk or cpu config for different components,
            // so just use the default value.
            ByteAmount instanceDisk = instanceDiskDefault;
            double instanceCpu = instanceCpuDefault;
            Resource resource = new Resource(instanceCpu, instanceRam, instanceDisk);
            // Insert it into the map
            instancePlanMap.put(instanceId, new PackingPlan.InstancePlan(instanceId, resource));
            containerRam = containerRam.plus(instanceRam);
        }
        Resource resource = new Resource(containerCpu, containerRam, containerDiskInBytes);
        PackingPlan.ContainerPlan containerPlan = new PackingPlan.ContainerPlan(containerId, new HashSet<>(instancePlanMap.values()), resource);
        containerPlans.add(containerPlan);
    }
    PackingPlan plan = new PackingPlan(topology.getId(), containerPlans);
    // Check whether it is a valid PackingPlan
    validatePackingPlan(plan);
    return plan;
}

Example 60 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project heron by twitter.

the class PackingUtils method computeTotalResourceChange.

/**
   * Identifies the resources reclaimed by the components that will be scaled down
   *
   * @return Total resources reclaimed
   */
public static Resource computeTotalResourceChange(TopologyAPI.Topology topology, Map<String, Integer> componentChanges, Resource defaultInstanceResources, ScalingDirection scalingDirection) {
    double cpu = 0;
    ByteAmount ram = ByteAmount.ZERO;
    ByteAmount disk = ByteAmount.ZERO;
    Map<String, ByteAmount> ramMap = TopologyUtils.getComponentRamMapConfig(topology);
    Map<String, Integer> componentsToScale = PackingUtils.getComponentsToScale(componentChanges, scalingDirection);
    for (String component : componentsToScale.keySet()) {
        int parallelismChange = Math.abs(componentChanges.get(component));
        cpu += parallelismChange * defaultInstanceResources.getCpu();
        disk = disk.plus(defaultInstanceResources.getDisk().multiply(parallelismChange));
        if (ramMap.containsKey(component)) {
            ram = ram.plus(ramMap.get(component).multiply(parallelismChange));
        } else {
            ram = ram.plus(defaultInstanceResources.getRam().multiply(parallelismChange));
        }
    }
    return new Resource(cpu, ram, disk);
}