use of com.twitter.heron.spi.common.Config in project heron by twitter.
the class RoundRobinPackingTest method testEvenPacking.
/**
* test even packing of instances
*/
@Test
public void testEvenPacking() throws Exception {
int numContainers = 2;
int componentParallelism = 4;
// Set up the topology and its config
com.twitter.heron.api.Config topologyConfig = new com.twitter.heron.api.Config();
topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
TopologyAPI.Topology topology = getTopology(componentParallelism, componentParallelism, topologyConfig);
int numInstance = TopologyUtils.getTotalInstance(topology);
// Two components
Assert.assertEquals(2 * componentParallelism, numInstance);
PackingPlan output = getRoundRobinPackingPlan(topology);
Assert.assertEquals(numContainers, output.getContainers().size());
Assert.assertEquals((Integer) numInstance, output.getInstanceCount());
for (PackingPlan.ContainerPlan container : output.getContainers()) {
Assert.assertEquals(numInstance / numContainers, container.getInstances().size());
// Verify each container got 2 spout and 2 bolt and container 1 got
assertComponentCount(container, "spout", 2);
assertComponentCount(container, "bolt", 2);
}
}
use of com.twitter.heron.spi.common.Config in project heron by twitter.
the class RoundRobinPackingTest method testPartialRamMap.
/**
* Test the scenario ram map config is partially set
*/
@Test
public void testPartialRamMap() throws Exception {
int numContainers = 2;
int spoutParallelism = 4;
int boltParallelism = 3;
Integer totalInstances = spoutParallelism + boltParallelism;
// Set up the topology and its config
com.twitter.heron.api.Config topologyConfig = new com.twitter.heron.api.Config();
topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
// Explicit set resources for container
ByteAmount containerRam = ByteAmount.fromGigabytes(10);
// Explicit set component ram map
ByteAmount boltRam = ByteAmount.fromGigabytes(1);
topologyConfig.setContainerRamRequested(containerRam);
topologyConfig.setComponentRam(BOLT_NAME, boltRam);
TopologyAPI.Topology topologyExplicitRamMap = getTopology(spoutParallelism, boltParallelism, topologyConfig);
PackingPlan packingPlanExplicitRamMap = getRoundRobinPackingPlan(topologyExplicitRamMap);
Assert.assertEquals(totalInstances, packingPlanExplicitRamMap.getInstanceCount());
// Ram for bolt should be the value in component ram map
for (PackingPlan.ContainerPlan containerPlan : packingPlanExplicitRamMap.getContainers()) {
Assert.assertEquals(containerRam, containerPlan.getRequiredResource().getRam());
int boltCount = 0;
int instancesCount = containerPlan.getInstances().size();
for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
if (instancePlan.getComponentName().equals(BOLT_NAME)) {
Assert.assertEquals(boltRam, instancePlan.getResource().getRam());
boltCount++;
}
}
// Ram for spout should be:
// (containerRam - all ram for bolt - ram for padding) / (# of spouts)
int spoutCount = instancesCount - boltCount;
for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
if (instancePlan.getComponentName().equals(SPOUT_NAME)) {
Assert.assertEquals(containerRam.minus(boltRam.multiply(boltCount)).minus(RoundRobinPacking.DEFAULT_RAM_PADDING_PER_CONTAINER).divide(spoutCount), instancePlan.getResource().getRam());
}
}
}
}
use of com.twitter.heron.spi.common.Config in project heron by twitter.
the class RoundRobinPackingTest method testContainerRequestedResources.
/**
* Test the scenario container level resource config are set
*/
@Test
public void testContainerRequestedResources() throws Exception {
int numContainers = 2;
int spoutParallelism = 4;
int boltParallelism = 3;
Integer totalInstances = spoutParallelism + boltParallelism;
// Set up the topology and its config
com.twitter.heron.api.Config topologyConfig = new com.twitter.heron.api.Config();
topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
// Explicit set resources for container
ByteAmount containerRam = ByteAmount.fromGigabytes(10);
ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
float containerCpu = 30;
topologyConfig.setContainerRamRequested(containerRam);
topologyConfig.setContainerDiskRequested(containerDisk);
topologyConfig.setContainerCpuRequested(containerCpu);
TopologyAPI.Topology topologyExplicitResourcesConfig = getTopology(spoutParallelism, boltParallelism, topologyConfig);
PackingPlan packingPlanExplicitResourcesConfig = getRoundRobinPackingPlan(topologyExplicitResourcesConfig);
Assert.assertEquals(numContainers, packingPlanExplicitResourcesConfig.getContainers().size());
Assert.assertEquals(totalInstances, packingPlanExplicitResourcesConfig.getInstanceCount());
for (PackingPlan.ContainerPlan containerPlan : packingPlanExplicitResourcesConfig.getContainers()) {
Assert.assertEquals(containerCpu, containerPlan.getRequiredResource().getCpu(), DELTA);
Assert.assertTrue(// due to round-off when using divide()
String.format("expected: %s but was: %s", containerRam, containerPlan.getRequiredResource().getRam()), Math.abs(containerRam.minus(containerPlan.getRequiredResource().getRam()).asBytes()) <= 1);
Assert.assertEquals(containerDisk, containerPlan.getRequiredResource().getDisk());
// All instances' resource requirement should be equal
// So the size of set should be 1
Set<Resource> resources = new HashSet<>();
for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
resources.add(instancePlan.getResource());
}
Assert.assertEquals(1, resources.size());
int instancesCount = containerPlan.getInstances().size();
Assert.assertEquals(containerRam.minus(RoundRobinPacking.DEFAULT_RAM_PADDING_PER_CONTAINER).divide(instancesCount), resources.iterator().next().getRam());
}
}
use of com.twitter.heron.spi.common.Config in project heron by twitter.
the class RoundRobinPackingTest method testCompleteRamMapRequested.
/**
* Test the scenario ram map config is partially set
*/
@Test
public void testCompleteRamMapRequested() throws Exception {
int numContainers = 2;
int spoutParallelism = 4;
int boltParallelism = 3;
Integer totalInstances = spoutParallelism + boltParallelism;
// Set up the topology and its config
com.twitter.heron.api.Config topologyConfig = new com.twitter.heron.api.Config();
topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
// Explicit set resources for container
// the value should be ignored, since we set the complete component ram map
ByteAmount containerRam = ByteAmount.fromBytes(Long.MAX_VALUE);
// Explicit set component ram map
ByteAmount boltRam = ByteAmount.fromGigabytes(1);
ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
topologyConfig.setContainerRamRequested(containerRam);
topologyConfig.setComponentRam(BOLT_NAME, boltRam);
topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
TopologyAPI.Topology topologyExplicitRamMap = getTopology(spoutParallelism, boltParallelism, topologyConfig);
PackingPlan packingPlanExplicitRamMap = getRoundRobinPackingPlan(topologyExplicitRamMap);
AssertPacking.assertContainers(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, SPOUT_NAME, boltRam, spoutRam, containerRam);
Assert.assertEquals(totalInstances, packingPlanExplicitRamMap.getInstanceCount());
}
use of com.twitter.heron.spi.common.Config in project heron by twitter.
the class MetricsCacheManagerHttpServer method main.
/**
* manual test for local mode topology, for debug
* How to run:
* in the [source root directory], run bazel test,
* bazel run heron/metricscachemgr/src/java:metricscache-queryclient-unshaded -- \
* <topology_name> <component_name> <metrics_name>
* Example:
* 1. run the example topology,
* ~/bin/heron submit local ~/.heron/examples/heron-examples.jar \
* com.twitter.heron.examples.ExclamationTopology ExclamationTopology \
* --deploy-deactivated --verbose
* 2. in the [source root directory],
* bazel run heron/metricscachemgr/src/java:metricscache-queryclient-unshaded -- \
* ExclamationTopology exclaim1 \
* __emit-count __execute-count __fail-count __ack-count __complete-latency __execute-latency \
* __process-latency __jvm-uptime-secs __jvm-process-cpu-load __jvm-memory-used-mb \
* __jvm-memory-mb-total __jvm-gc-collection-time-ms __server/__time_spent_back_pressure_initiated \
* __time_spent_back_pressure_by_compid
*/
public static void main(String[] args) throws ExecutionException, InterruptedException, IOException {
if (args.length < 3) {
System.out.println("Usage: java MetricsQuery <topology_name> <component_name> <metrics_name>");
} else {
System.out.println("topology: " + args[0] + "; component: " + args[1]);
}
Config config = Config.newBuilder().put(Key.STATEMGR_ROOT_PATH, System.getProperty("user.home") + "/.herondata/repository/state/local").build();
LocalFileSystemStateManager stateManager = new LocalFileSystemStateManager();
stateManager.initialize(config);
TopologyMaster.MetricsCacheLocation location = stateManager.getMetricsCacheLocation(null, args[0]).get();
if (location == null || !location.isInitialized()) {
System.out.println("MetricsCacheMgr is not ready");
return;
}
// construct metric cache stat url
String url = "http://" + location.getHost() + ":" + location.getStatsPort() + MetricsCacheManagerHttpServer.PATH_STATS;
// construct query payload
byte[] requestData = TopologyMaster.MetricRequest.newBuilder().setComponentName(args[1]).setMinutely(true).setInterval(-1).addAllMetric(Arrays.asList(Arrays.copyOfRange(args, 2, args.length))).build().toByteArray();
// http communication
HttpURLConnection con = NetworkUtils.getHttpConnection(url);
NetworkUtils.sendHttpPostRequest(con, "X", requestData);
byte[] responseData = NetworkUtils.readHttpResponse(con);
// parse response data
TopologyMaster.MetricResponse response = TopologyMaster.MetricResponse.parseFrom(responseData);
System.out.println(response.toString());
}
Aggregations