Examples with ByteAmount com.twitter.heron.common.basics.ByteAmount used on opensource projects

Example 51 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class Resource method plus.

/**
 * Adds a given resource from the current resource.
 */
public Resource plus(Resource other) {
    double totalCpu = this.getCpu() + other.getCpu();
    ByteAmount totalRam = this.getRam().plus(other.getRam());
    ByteAmount totalDisk = this.getDisk().plus(other.getDisk());
    return new Resource(totalCpu, totalRam, totalDisk);
}

Example 52 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class SpoutInstance method produceTuple.

protected void produceTuple() {
    int maxSpoutPending = TypeUtils.getInteger(config.get(Config.TOPOLOGY_MAX_SPOUT_PENDING));
    long totalTuplesEmitted = collector.getTotalTuplesEmitted();
    long totalBytesEmitted = collector.getTotalBytesEmitted();
    Duration instanceEmitBatchTime = systemConfig.getInstanceEmitBatchTime();
    ByteAmount instanceEmitBatchSize = systemConfig.getInstanceEmitBatchSize();
    long startOfCycle = System.nanoTime();
    // We would reuse the System.nanoTime()
    long currentTime = startOfCycle;
    while (!ackEnabled || (maxSpoutPending > collector.numInFlight())) {
        // Delegate to the use defined spout
        spout.nextTuple();
        // Swap
        long startTime = currentTime;
        currentTime = System.nanoTime();
        long latency = currentTime - startTime;
        spoutMetrics.nextTuple(latency);
        long newTotalTuplesEmitted = collector.getTotalTuplesEmitted();
        long newTotalBytesEmitted = collector.getTotalBytesEmitted();
        if (newTotalTuplesEmitted == totalTuplesEmitted) {
            // No tuples to emit....
            break;
        }
        totalTuplesEmitted = newTotalTuplesEmitted;
        // To avoid spending too much time
        if (currentTime - startOfCycle - instanceEmitBatchTime.toNanos() > 0) {
            break;
        }
        if (!ByteAmount.fromBytes(newTotalBytesEmitted - totalBytesEmitted).lessThan(instanceEmitBatchSize)) {
            break;
        }
    }
}

Example 53 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class TopologyUtilsTest method testGetComponentDiskMapAllDiskSpecified.

@Test
public void testGetComponentDiskMapAllDiskSpecified() {
    int componentParallelism = 2;
    Config topologyConfig = new Config();
    Map<String, Integer> spouts = new HashMap<>();
    spouts.put("spout", componentParallelism);
    Map<String, Integer> bolts = new HashMap<>();
    bolts.put("bolt", componentParallelism);
    ByteAmount boltDisk = ByteAmount.fromGigabytes(1);
    ByteAmount spoutDisk = ByteAmount.fromGigabytes(2);
    topologyConfig.setComponentDisk("spout", spoutDisk);
    topologyConfig.setComponentDisk("bolt", boltDisk);
    // sort the component disk map
    Map<String, ByteAmount> diskMap = new TreeMap<>(TopologyUtils.getComponentDiskMapConfig(TopologyTests.createTopology("test", topologyConfig, spouts, bolts)));
    Assert.assertArrayEquals(new String[] { "bolt", "spout" }, diskMap.keySet().toArray());
    Assert.assertArrayEquals(new ByteAmount[] { boltDisk, spoutDisk }, diskMap.values().toArray());
}

Example 54 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class TopologyUtilsTest method testGetComponentRamMapAllRamSpecified.

@Test
public void testGetComponentRamMapAllRamSpecified() {
    int componentParallelism = 2;
    Config topologyConfig = new Config();
    Map<String, Integer> spouts = new HashMap<>();
    spouts.put("spout", componentParallelism);
    Map<String, Integer> bolts = new HashMap<>();
    bolts.put("bolt", componentParallelism);
    ByteAmount boltRam = ByteAmount.fromGigabytes(1);
    ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setComponentRam("spout", spoutRam);
    topologyConfig.setComponentRam("bolt", boltRam);
    // sort the component ram map
    Map<String, ByteAmount> ramMap = new TreeMap<>(TopologyUtils.getComponentRamMapConfig(TopologyTests.createTopology("test", topologyConfig, spouts, bolts)));
    Assert.assertArrayEquals(new String[] { "bolt", "spout" }, ramMap.keySet().toArray());
    Assert.assertArrayEquals(new ByteAmount[] { boltRam, spoutRam }, ramMap.values().toArray());
}

Example 55 with ByteAmount

use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.

the class TopologyUtilsTest method testGetComponentRamMapSomeRamSpecified.

@Test
public void testGetComponentRamMapSomeRamSpecified() {
    int componentParallelism = 2;
    Config topologyConfig = new Config();
    Map<String, Integer> spouts = new HashMap<>();
    spouts.put("spout", componentParallelism);
    Map<String, Integer> bolts = new HashMap<>();
    bolts.put("bolt", componentParallelism);
    ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setComponentRam("spout", spoutRam);
    // sort the component ram map
    Map<String, ByteAmount> ramMap = new TreeMap<>(TopologyUtils.getComponentRamMapConfig(TopologyTests.createTopology("test", topologyConfig, spouts, bolts)));
    // Component ram map sets only spout's ram
    Assert.assertArrayEquals(new String[] { "spout" }, ramMap.keySet().toArray());
    Assert.assertArrayEquals(new ByteAmount[] { spoutRam }, ramMap.values().toArray());
}