use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.
the class Resource method plus.
/**
* Adds a given resource from the current resource.
*/
public Resource plus(Resource other) {
double totalCpu = this.getCpu() + other.getCpu();
ByteAmount totalRam = this.getRam().plus(other.getRam());
ByteAmount totalDisk = this.getDisk().plus(other.getDisk());
return new Resource(totalCpu, totalRam, totalDisk);
}
use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.
the class SpoutInstance method produceTuple.
protected void produceTuple() {
int maxSpoutPending = TypeUtils.getInteger(config.get(Config.TOPOLOGY_MAX_SPOUT_PENDING));
long totalTuplesEmitted = collector.getTotalTuplesEmitted();
long totalBytesEmitted = collector.getTotalBytesEmitted();
Duration instanceEmitBatchTime = systemConfig.getInstanceEmitBatchTime();
ByteAmount instanceEmitBatchSize = systemConfig.getInstanceEmitBatchSize();
long startOfCycle = System.nanoTime();
// We would reuse the System.nanoTime()
long currentTime = startOfCycle;
while (!ackEnabled || (maxSpoutPending > collector.numInFlight())) {
// Delegate to the use defined spout
spout.nextTuple();
// Swap
long startTime = currentTime;
currentTime = System.nanoTime();
long latency = currentTime - startTime;
spoutMetrics.nextTuple(latency);
long newTotalTuplesEmitted = collector.getTotalTuplesEmitted();
long newTotalBytesEmitted = collector.getTotalBytesEmitted();
if (newTotalTuplesEmitted == totalTuplesEmitted) {
// No tuples to emit....
break;
}
totalTuplesEmitted = newTotalTuplesEmitted;
// To avoid spending too much time
if (currentTime - startOfCycle - instanceEmitBatchTime.toNanos() > 0) {
break;
}
if (!ByteAmount.fromBytes(newTotalBytesEmitted - totalBytesEmitted).lessThan(instanceEmitBatchSize)) {
break;
}
}
}
use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.
the class TopologyUtilsTest method testGetComponentDiskMapAllDiskSpecified.
@Test
public void testGetComponentDiskMapAllDiskSpecified() {
int componentParallelism = 2;
Config topologyConfig = new Config();
Map<String, Integer> spouts = new HashMap<>();
spouts.put("spout", componentParallelism);
Map<String, Integer> bolts = new HashMap<>();
bolts.put("bolt", componentParallelism);
ByteAmount boltDisk = ByteAmount.fromGigabytes(1);
ByteAmount spoutDisk = ByteAmount.fromGigabytes(2);
topologyConfig.setComponentDisk("spout", spoutDisk);
topologyConfig.setComponentDisk("bolt", boltDisk);
// sort the component disk map
Map<String, ByteAmount> diskMap = new TreeMap<>(TopologyUtils.getComponentDiskMapConfig(TopologyTests.createTopology("test", topologyConfig, spouts, bolts)));
Assert.assertArrayEquals(new String[] { "bolt", "spout" }, diskMap.keySet().toArray());
Assert.assertArrayEquals(new ByteAmount[] { boltDisk, spoutDisk }, diskMap.values().toArray());
}
use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.
the class TopologyUtilsTest method testGetComponentRamMapAllRamSpecified.
@Test
public void testGetComponentRamMapAllRamSpecified() {
int componentParallelism = 2;
Config topologyConfig = new Config();
Map<String, Integer> spouts = new HashMap<>();
spouts.put("spout", componentParallelism);
Map<String, Integer> bolts = new HashMap<>();
bolts.put("bolt", componentParallelism);
ByteAmount boltRam = ByteAmount.fromGigabytes(1);
ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
topologyConfig.setComponentRam("spout", spoutRam);
topologyConfig.setComponentRam("bolt", boltRam);
// sort the component ram map
Map<String, ByteAmount> ramMap = new TreeMap<>(TopologyUtils.getComponentRamMapConfig(TopologyTests.createTopology("test", topologyConfig, spouts, bolts)));
Assert.assertArrayEquals(new String[] { "bolt", "spout" }, ramMap.keySet().toArray());
Assert.assertArrayEquals(new ByteAmount[] { boltRam, spoutRam }, ramMap.values().toArray());
}
use of com.twitter.heron.common.basics.ByteAmount in project incubator-heron by apache.
the class TopologyUtilsTest method testGetComponentRamMapSomeRamSpecified.
@Test
public void testGetComponentRamMapSomeRamSpecified() {
int componentParallelism = 2;
Config topologyConfig = new Config();
Map<String, Integer> spouts = new HashMap<>();
spouts.put("spout", componentParallelism);
Map<String, Integer> bolts = new HashMap<>();
bolts.put("bolt", componentParallelism);
ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
topologyConfig.setComponentRam("spout", spoutRam);
// sort the component ram map
Map<String, ByteAmount> ramMap = new TreeMap<>(TopologyUtils.getComponentRamMapConfig(TopologyTests.createTopology("test", topologyConfig, spouts, bolts)));
// Component ram map sets only spout's ram
Assert.assertArrayEquals(new String[] { "spout" }, ramMap.keySet().toArray());
Assert.assertArrayEquals(new ByteAmount[] { spoutRam }, ramMap.values().toArray());
}
Aggregations