Examples with PackingPlan org.apache.heron.spi.packing.PackingPlan used on opensource projects

Example 11 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class FirstFitDecreasingPackingTest method testContainerRequestedResources.

/**
 * Test the scenario where container level resource config are set
 */
@Test
public void testContainerRequestedResources() throws Exception {
    // Explicit set resources for container
    ByteAmount containerRam = ByteAmount.fromGigabytes(10);
    ByteAmount containerDisk = ByteAmount.fromGigabytes(20);
    double containerCpu = 30;
    Resource containerResource = new Resource(containerCpu, containerRam, containerDisk);
    Resource padding = PackingUtils.finalizePadding(new Resource(containerCpu, containerRam, containerDisk), new Resource(PackingUtils.DEFAULT_CONTAINER_CPU_PADDING, PackingUtils.DEFAULT_CONTAINER_RAM_PADDING, PackingUtils.DEFAULT_CONTAINER_RAM_PADDING), PackingUtils.DEFAULT_CONTAINER_PADDING_PERCENTAGE);
    topologyConfig.setContainerRamRequested(containerRam);
    topologyConfig.setContainerDiskRequested(containerDisk);
    topologyConfig.setContainerCpuRequested(containerCpu);
    topology = getTopology(spoutParallelism, boltParallelism, topologyConfig);
    PackingPlan packingPlan = doPackingTest(topology, instanceDefaultResources, boltParallelism, instanceDefaultResources, spoutParallelism, 2, containerResource);
    for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
        int instanceCount = containerPlan.getInstances().size();
        Assert.assertEquals(Math.round(instanceCount * instanceDefaultResources.getCpu() + padding.getCpu()), (long) containerPlan.getRequiredResource().getCpu());
        Assert.assertEquals(instanceDefaultResources.getRam().multiply(instanceCount).plus(padding.getRam()), containerPlan.getRequiredResource().getRam());
        Assert.assertEquals(instanceDefaultResources.getDisk().multiply(instanceCount).plus(padding.getDisk()), containerPlan.getRequiredResource().getDisk());
        // All instances' resource requirement should be equal
        // So the size of set should be 1
        Set<Resource> resources = new HashSet<>();
        for (PackingPlan.InstancePlan instancePlan : containerPlan.getInstances()) {
            resources.add(instancePlan.getResource());
        }
        Assert.assertEquals(1, resources.size());
        Assert.assertEquals(instanceDefaultResources.getRam(), resources.iterator().next().getRam());
    }
}

Example 12 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class V1Controller method submit.

/**
 * Configures all components required by a <code>topology</code> and submits it to the Kubernetes scheduler.
 * @param packingPlan Used to configure the StatefulSets <code>Resource</code>s and replica count.
 * @return Success indicator.
 */
@Override
boolean submit(PackingPlan packingPlan) {
    final String topologyName = getTopologyName();
    if (!topologyName.equals(topologyName.toLowerCase())) {
        throw new TopologySubmissionException("K8S scheduler does not allow upper case topology's.");
    }
    final Resource containerResource = getContainerResource(packingPlan);
    final V1Service topologyService = createTopologyService();
    try {
        coreClient.createNamespacedService(getNamespace(), topologyService, null, null, null);
    } catch (ApiException e) {
        KubernetesUtils.logExceptionWithDetails(LOG, "Error creating topology service", e);
        throw new TopologySubmissionException(e.getMessage());
    }
    // Find the max number of instances in a container so that we can open
    // enough ports if remote debugging is enabled.
    int numberOfInstances = 0;
    for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
        numberOfInstances = Math.max(numberOfInstances, containerPlan.getInstances().size());
    }
    final V1StatefulSet executors = createStatefulSet(containerResource, numberOfInstances, true);
    final V1StatefulSet manager = createStatefulSet(containerResource, numberOfInstances, false);
    try {
        appsClient.createNamespacedStatefulSet(getNamespace(), executors, null, null, null);
        appsClient.createNamespacedStatefulSet(getNamespace(), manager, null, null, null);
    } catch (ApiException e) {
        final String message = String.format("Error creating topology: %s%n", e.getResponseBody());
        KubernetesUtils.logExceptionWithDetails(LOG, message, e);
        throw new TopologySubmissionException(message);
    }
    return true;
}

Example 13 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class LauncherUtilsTest method constructsRuntimeWithPackingProperly.

@Test
public void constructsRuntimeWithPackingProperly() {
    Config runtime = Config.newBuilder().put("key-23", "value-34").build();
    Assert.assertNull(Runtime.componentRamMap(runtime));
    Set<PackingPlan.ContainerPlan> containers = new HashSet<>();
    containers.add(Mockito.mock(PackingPlan.ContainerPlan.class));
    containers.add(Mockito.mock(PackingPlan.ContainerPlan.class));
    PackingPlan mockPacking = Mockito.mock(PackingPlan.class);
    Mockito.when(mockPacking.getComponentRamDistribution()).thenReturn("ramMap");
    Mockito.when(mockPacking.getContainers()).thenReturn(containers);
    Config newRuntime = LauncherUtils.getInstance().createConfigWithPackingDetails(runtime, mockPacking);
    Assert.assertNull(Runtime.componentRamMap(runtime));
    Assert.assertEquals("ramMap", Runtime.componentRamMap(newRuntime));
    Assert.assertEquals(3, Runtime.numContainers(newRuntime).longValue());
    Assert.assertEquals("value-34", newRuntime.getStringValue("key-23"));
}

Example 14 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class LauncherUtilsTest method generatesPackingPlan.

@Test
public void generatesPackingPlan() throws Exception {
    final String PACKING_CLASS = "nonExistingTestPackingClass";
    final PackingPlan mockPackingPlan = Mockito.mock(PackingPlan.class);
    IPacking mockPacking = Mockito.mock(IPacking.class);
    Mockito.when(mockPacking.pack()).thenReturn(mockPackingPlan);
    PowerMockito.spy(ReflectionUtils.class);
    PowerMockito.doReturn(mockPacking).when(ReflectionUtils.class, "newInstance", PACKING_CLASS);
    TopologyAPI.Topology mockTopology = PowerMockito.mock(TopologyAPI.Topology.class);
    Config mockConfig = Mockito.mock(Config.class);
    Mockito.when(mockConfig.getStringValue(Key.PACKING_CLASS)).thenReturn(PACKING_CLASS);
    Mockito.when(mockConfig.get(Key.TOPOLOGY_DEFINITION)).thenReturn(mockTopology);
    PackingPlan resultPacking = LauncherUtils.getInstance().createPackingPlan(mockConfig, mockConfig);
    Assert.assertEquals(mockPackingPlan, resultPacking);
    Mockito.verify(mockPacking).initialize(Mockito.any(Config.class), Mockito.eq(mockTopology));
    Mockito.verify(mockPacking).pack();
    Mockito.verify(mockPacking).close();
}

Example 15 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class LocalSchedulerTest method testOnSchedule.

@Test
public void testOnSchedule() throws Exception {
    Mockito.doNothing().when(scheduler).startExecutorMonitor(Mockito.anyInt(), Mockito.any(Process.class), Mockito.anySet());
    Process[] mockProcesses = new Process[4];
    for (int i = 0; i < 4; i++) {
        mockProcesses[i] = Mockito.mock(Process.class);
        Set<PackingPlan.InstancePlan> instances = (i == 0) ? null : PackingTestUtils.testContainerPlan(i).getInstances();
        Mockito.doReturn(mockProcesses[i]).when(scheduler).startExecutorProcess(i, instances);
    }
    PackingPlan packingPlan = Mockito.mock(PackingPlan.class);
    Set<PackingPlan.ContainerPlan> containers = new HashSet<>();
    containers.add(PackingTestUtils.testContainerPlan(1));
    containers.add(PackingTestUtils.testContainerPlan(3));
    Mockito.when(packingPlan.getContainers()).thenReturn(containers);
    Assert.assertTrue(scheduler.onSchedule(packingPlan));
    verifyIdsOfLaunchedContainers(0, 1, 3);
    for (int i = 0; i < 4; i++) {
        if (i == 2) {
            // id 2 was not in the container plan
            continue;
        }
        Set<PackingPlan.InstancePlan> instances = (i == 0) ? null : PackingTestUtils.testContainerPlan(i).getInstances();
        Mockito.verify(scheduler).startExecutor(i, instances);
        Mockito.verify(scheduler).startExecutorProcess(i, instances);
        Mockito.verify(scheduler).startExecutorMonitor(i, mockProcesses[i], instances);
    }
}