Examples with ByteAmount org.apache.heron.common.basics.ByteAmount used on opensource projects

Example 31 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class ConfigBuilder method setComponentLevelResource.

private void setComponentLevelResource(Config config, Map.Entry<String, Object> entry) {
    List<Object> objects = (List<Object>) entry.getValue();
    for (Object obj : objects) {
        String objString = obj.toString();
        objString = objString.replace(COMMA, WHITESPACE);
        objString = objString.replace(LEFT_BRACE, WHITESPACE);
        objString = objString.replace(RIGHT_BRACE, WHITESPACE);
        int idIndex = objString.indexOf(ID);
        int ramIndex = objString.indexOf(RAM);
        int cpuIndex = objString.indexOf(CPU);
        int diskIndex = objString.indexOf(DISK);
        String ramWithUom = "";
        String id = getIdValue(objString, idIndex);
        // todo (josh fischer) diskWithUom and cpu are still to be implemented for use with k8s
        String diskWithUom = "";
        String cpu = "";
        if (ramIndex != -1) {
            ramWithUom = assignValue(objString, ramIndex);
        }
        if (cpuIndex != -1) {
            cpu = assignValue(objString, cpuIndex);
        }
        if (diskIndex != -1) {
            diskWithUom = assignValue(objString, diskIndex);
        }
        ByteAmount byteAmount = null;
        if (ramWithUom.contains(MB)) {
            // its megaBytes
            int mbIndex = verifyStartingIndexOfUom(ramWithUom, MB);
            long megaBytes = extractRawValue(ramWithUom, mbIndex);
            if (megaBytes < MINIMUM_MB) {
                throw new IllegalArgumentException("The minimum RAM resource allocation for a component must be at least 256MB");
            }
            byteAmount = ByteAmount.fromMegabytes(megaBytes);
        } else if (ramWithUom.contains(GB)) {
            // its gigaBytes
            // we don't validate here as NumberFormatException is thrown converting decimals to longs
            int gbIndex = verifyStartingIndexOfUom(ramWithUom, GB);
            byteAmount = ByteAmount.fromGigabytes(extractRawValue(ramWithUom, gbIndex));
        } else if (ramWithUom.contains(B)) {
            // its bytes
            int bIndex = verifyStartingIndexOfUom(ramWithUom, B);
            long bytes = extractRawValue(ramWithUom, bIndex);
            if (bytes < MINIMUM_BYTES) {
                throw new IllegalArgumentException("The minimum RAM resource allocation for a component must be at least 256000000B");
            }
            byteAmount = ByteAmount.fromBytes(bytes);
        } else {
            // There is no format throw an exception
            throw new IllegalArgumentException(" Please specify 'B', 'MB', 'GB' when declaring RAM Resources");
        }
        config.setComponentRam(id, byteAmount);
    }
}

Example 32 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class TopologyUtilsTest method testGetComponentRamMapDefaultValue.

@Test
public void testGetComponentRamMapDefaultValue() {
    int componentParallelism = 2;
    Config topologyConfig = new Config();
    Map<String, Integer> spouts = new HashMap<>();
    spouts.put("spout", componentParallelism);
    Map<String, Integer> bolts = new HashMap<>();
    bolts.put("bolt", componentParallelism);
    // sort the component RAM map
    Map<String, ByteAmount> ramMap = new TreeMap<>(TopologyUtils.getComponentRamMapConfig(TopologyTests.createTopology("test", topologyConfig, spouts, bolts)));
    // Component RAM map is not set, the ramMap size should be 0
    Assert.assertEquals(0, ramMap.size());
}

Example 33 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class JVMMetrics method updateBufferPoolMetrics.

// Gather metrics related to both direct and mapped byte buffers in the jvm.
// These metrics can be useful for diagnosing native memory usage.
private void updateBufferPoolMetrics() {
    for (BufferPoolMXBean bufferPoolMXBean : bufferPoolMXBeanList) {
        String normalizedKeyName = bufferPoolMXBean.getName().replaceAll("[^\\w]", "-");
        final ByteAmount memoryUsed = ByteAmount.fromBytes(bufferPoolMXBean.getMemoryUsed());
        final ByteAmount totalCapacity = ByteAmount.fromBytes(bufferPoolMXBean.getTotalCapacity());
        final ByteAmount count = ByteAmount.fromBytes(bufferPoolMXBean.getCount());
        // The estimated memory the JVM is using for this buffer pool
        jvmBufferPoolMemoryUsage.safeScope(normalizedKeyName + "-memory-used").setValue(memoryUsed.asMegabytes());
        // The estimated total capacity of the buffers in this pool
        jvmBufferPoolMemoryUsage.safeScope(normalizedKeyName + "-total-capacity").setValue(totalCapacity.asMegabytes());
        // THe estimated number of buffers in this pool
        jvmBufferPoolMemoryUsage.safeScope(normalizedKeyName + "-count").setValue(count.asMegabytes());
    }
}

Example 34 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class HeronMasterDriver method findLargestFittingWorker.

/**
 * YARN allocates resources in fixed increments of memory and CPU. As a result, the actual
 * resource allocation may be little more than what was requested. This method finds the biggest
 * heron containerPlan that will fit the allocated YARN container. In some cases the YARN CPU
 * scheduling may be disabled, resulting in default core allocation to each container. This
 * method can ignore core fitting in such a case.
 */
@VisibleForTesting
Optional<HeronWorker> findLargestFittingWorker(AllocatedEvaluator evaluator, Collection<HeronWorker> pendingWorkers, boolean ignoreCpu) {
    ByteAmount allocatedRam = ByteAmount.fromMegabytes(evaluator.getEvaluatorDescriptor().getMemory());
    int allocatedCores = evaluator.getEvaluatorDescriptor().getNumberOfCores();
    HeronWorker biggestFittingWorker = null;
    for (HeronWorker worker : pendingWorkers) {
        if (worker.mem.greaterThan(allocatedRam)) {
            continue;
        }
        if (!ignoreCpu) {
            if (worker.cores > allocatedCores) {
                continue;
            }
        }
        if (biggestFittingWorker != null) {
            if (worker.mem.lessThan(biggestFittingWorker.mem) || worker.cores < biggestFittingWorker.cores) {
                continue;
            }
        }
        biggestFittingWorker = worker;
    }
    return Optional.fromNullable(biggestFittingWorker);
}

Example 35 with ByteAmount

use of org.apache.heron.common.basics.ByteAmount in project heron by twitter.

the class MesosScheduler method fillResourcesRequirementForBaseContainer.

/**
 * Fill the the resources requirement, i.e. CPU, memory and disk for the given container.
 * This method changes the BaseContainer passed in.
 * <p>
 * Notice: Currently we just make every container homogeneous,
 * requiring maximum resources for every container.
 *
 * @param container the BaseContainer to fill value in
 * @param containerIndex the index of the container
 * @param packing the packing plan
 */
protected void fillResourcesRequirementForBaseContainer(BaseContainer container, Integer containerIndex, PackingPlan packing) {
    PackingPlan updatedPackingPlan = packing.cloneWithHomogeneousScheduledResource();
    Resource maxResourceContainer = updatedPackingPlan.getContainers().iterator().next().getRequiredResource();
    double cpu = 0;
    ByteAmount disk = ByteAmount.ZERO;
    ByteAmount mem = ByteAmount.ZERO;
    for (PackingPlan.ContainerPlan cp : packing.getContainers()) {
        Resource containerResource = cp.getRequiredResource();
        cpu = Math.max(cpu, containerResource.getCpu());
        disk = disk.max(containerResource.getDisk());
        mem = mem.max(containerResource.getRam());
    }
    container.cpu = maxResourceContainer.getCpu();
    // Convert them from bytes to MB
    container.diskInMB = maxResourceContainer.getDisk().asMegabytes();
    container.memInMB = maxResourceContainer.getRam().asMegabytes();
    container.ports = SchedulerUtils.ExecutorPort.getRequiredPorts().size();
}