Examples with Config com.twitter.heron.spi.common.Config used on opensource projects

Example 51 with Config

use of com.twitter.heron.spi.common.Config in project incubator-heron by apache.

the class RoundRobinPackingTest method testContainerRequestedResourcesWhenRamPaddingSet.

/**
 * Test the scenario container level resource config are set
 */
@Test
public void testContainerRequestedResourcesWhenRamPaddingSet() throws Exception {
    int numContainers = 2;
    int spoutParallelism = 4;
    int boltParallelism = 3;
    Integer totalInstances = spoutParallelism + boltParallelism;
    // Set up the topology and its config
    com.twitter.heron.api.Config topologyConfig = new com.twitter.heron.api.Config();
    topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(10);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    ByteAmount containerRamPadding = ByteAmount.fromMegabytes(512);
    double containerCpu = 30;
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    topologyConfig.setContainerRamPadding(containerRamPadding);
    TopologyAPI.Topology topologyExplicitResourcesConfig = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanExplicitResourcesConfig = getRoundRobinPackingPlan(topologyExplicitResourcesConfig);
    Assert.assertEquals(numContainers, packingPlanExplicitResourcesConfig.getContainers().size());
    Assert.assertEquals(totalInstances, packingPlanExplicitResourcesConfig.getInstanceCount());
    for (PackingPlan.ContainerPlan containerPlan : packingPlanExplicitResourcesConfig.getContainers()) {
        Assert.assertEquals(containerCpu, containerPlan.getRequiredResource().getCpu(), DELTA);
        Assert.assertTrue(// due to round-off when using divide()
        String.format("expected: %s but was: %s", containerRam, containerPlan.getRequiredResource().getRam()), Math.abs(containerRam.minus(containerPlan.getRequiredResource().getRam()).asBytes()) <= 1);
        Assert.assertEquals(containerDisk, containerPlan.getRequiredResource().getDisk());
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> resources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            resources.add(instancePlan.getResource());
        }
        Assert.assertEquals(1, resources.size());
        int instancesCount = containerPlan.getInstances().size();
        Assert.assertEquals(containerRam.minus(containerRamPadding).divide(instancesCount), resources.iterator().next().getRam());
    }
}

Example 52 with Config

use of com.twitter.heron.spi.common.Config in project incubator-heron by apache.

the class RoundRobinPackingTest method testEvenPacking.

/**
 * test even packing of instances
 */
@Test
public void testEvenPacking() throws Exception {
    int numContainers = 2;
    int componentParallelism = 4;
    // Set up the topology and its config
    com.twitter.heron.api.Config topologyConfig = new com.twitter.heron.api.Config();
    topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
    TopologyAPI.Topology topology = getTopology(componentParallelism, componentParallelism, topologyConfig);
    int numInstance = TopologyUtils.getTotalInstance(topology);
    // Two components
    Assert.assertEquals(2 * componentParallelism, numInstance);
    PackingPlan output = getRoundRobinPackingPlan(topology);
    Assert.assertEquals(numContainers, output.getContainers().size());
    Assert.assertEquals((Integer) numInstance, output.getInstanceCount());
    for (PackingPlan.ContainerPlan container : output.getContainers()) {
        Assert.assertEquals(numInstance / numContainers, container.getInstances().size());
        // Verify each container got 2 spout and 2 bolt and container 1 got
        assertComponentCount(container, "spout", 2);
        assertComponentCount(container, "bolt", 2);
    }
}

Example 53 with Config

use of com.twitter.heron.spi.common.Config in project incubator-heron by apache.

the class RoundRobinPackingTest method testCheckFailure.

@Test(expected = PackingException.class)
public void testCheckFailure() throws Exception {
    int numContainers = 2;
    int spoutParallelism = 4;
    int boltParallelism = 3;
    // Set up the topology and its config
    com.twitter.heron.api.Config topologyConfig = new com.twitter.heron.api.Config();
    topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
    // Explicit set insufficient ram for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(0);
    topologyConfig.setContainerRamRequested(containerRam);
    TopologyAPI.Topology topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    getRoundRobinPackingPlan(topology);
}

Example 54 with Config

use of com.twitter.heron.spi.common.Config in project incubator-heron by apache.

the class RoundRobinPackingTest method testCompleteRamMapRequested.

/**
 * Test the scenario ram map config is partially set
 */
@Test
public void testCompleteRamMapRequested() throws Exception {
    int numContainers = 2;
    int spoutParallelism = 4;
    int boltParallelism = 3;
    Integer totalInstances = spoutParallelism + boltParallelism;
    // Set up the topology and its config
    com.twitter.heron.api.Config topologyConfig = new com.twitter.heron.api.Config();
    topologyConfig.put(com.twitter.heron.api.Config.TOPOLOGY_STMGRS, numContainers);
    // Explicit set resources for container
    // the value should be ignored, since we set the complete component ram map
    ByteAmount containerRam = ByteAmount.fromBytes(Long.MAX_VALUE);
    // Explicit set component ram map
    ByteAmount boltRam = ByteAmount.fromGigabytes(1);
    ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setComponentRam(BOLT_NAME, boltRam);
    topologyConfig.setComponentRam(SPOUT_NAME, spoutRam);
    TopologyAPI.Topology topologyExplicitRamMap = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlanExplicitRamMap = getRoundRobinPackingPlan(topologyExplicitRamMap);
    AssertPacking.assertContainers(packingPlanExplicitRamMap.getContainers(), BOLT_NAME, SPOUT_NAME, boltRam, spoutRam, containerRam);
    Assert.assertEquals(totalInstances, packingPlanExplicitRamMap.getInstanceCount());
}

Example 55 with Config

use of com.twitter.heron.spi.common.Config in project incubator-heron by apache.

the class RuntimeManagerMain method manageTopology.

/**
 * Manager a topology
 * 1. Instantiate necessary resources
 * 2. Valid whether the runtime management is legal
 * 3. Complete the runtime management for a specific command
 */
public void manageTopology() throws TopologyRuntimeManagementException, TMasterException, PackingException {
    String topologyName = Context.topologyName(config);
    // 1. Do prepare work
    // create an instance of state manager
    String statemgrClass = Context.stateManagerClass(config);
    IStateManager statemgr;
    try {
        statemgr = ReflectionUtils.newInstance(statemgrClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new TopologyRuntimeManagementException(String.format("Failed to instantiate state manager class '%s'", statemgrClass), e);
    }
    // Put it in a try block so that we can always clean resources
    try {
        // initialize the statemgr
        statemgr.initialize(config);
        // TODO(mfu): timeout should read from config
        SchedulerStateManagerAdaptor adaptor = new SchedulerStateManagerAdaptor(statemgr, 5000);
        boolean hasExecutionData = validateRuntimeManage(adaptor, topologyName);
        // 2. Try to manage topology if valid
        // invoke the appropriate command to manage the topology
        LOG.log(Level.FINE, "Topology: {0} to be {1}ed", new Object[] { topologyName, command });
        // build the runtime config
        Config runtime = Config.newBuilder().put(Key.TOPOLOGY_NAME, Context.topologyName(config)).put(Key.SCHEDULER_STATE_MANAGER_ADAPTOR, adaptor).build();
        // Create a ISchedulerClient basing on the config
        ISchedulerClient schedulerClient = getSchedulerClient(runtime);
        callRuntimeManagerRunner(runtime, schedulerClient, !hasExecutionData);
    } finally {
        // 3. Do post work basing on the result
        // Currently nothing to do here
        // 4. Close the resources
        SysUtils.closeIgnoringExceptions(statemgr);
    }
}