Examples with Resource org.apache.heron.spi.packing.Resource used on opensource projects

Example 41 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class JsonFormatterUtils method renderInstancePlan.

ObjectNode renderInstancePlan(PackingPlan.InstancePlan instancePlan) {
    Resource resources = instancePlan.getResource();
    ObjectNode resourcesNode = mapper.createObjectNode();
    resourcesNode.put("cpu", resources.getCpu());
    resourcesNode.put("ram", resources.getRam().asBytes());
    resourcesNode.put("disk", resources.getDisk().asBytes());
    ObjectNode instancePlanNode = mapper.createObjectNode();
    instancePlanNode.put("component", instancePlan.getComponentName());
    instancePlanNode.put("id", instancePlan.getTaskId());
    instancePlanNode.put("index", instancePlan.getComponentIndex());
    instancePlanNode.set("resources", resourcesNode);
    return instancePlanNode;
}

Example 42 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class UpdateTopologyManagerTest method requestsToAddAndRemoveContainers.

/**
 * Test scalable scheduler invocation
 */
@Test
@PrepareForTest(TManagerUtils.class)
public void requestsToAddAndRemoveContainers() throws Exception {
    Lock lock = mockLock(true);
    SchedulerStateManagerAdaptor mockStateMgr = mockStateManager(testTopology, this.currentProtoPlan, lock);
    IScalable mockScheduler = mock(IScalable.class);
    HashSet<PackingPlan.ContainerPlan> mockRetrunSet = new HashSet<>();
    mockRetrunSet.add(new PackingPlan.ContainerPlan(0, new HashSet<>(), new Resource(5, ByteAmount.ZERO, ByteAmount.ZERO)));
    mockRetrunSet.add(new PackingPlan.ContainerPlan(1, new HashSet<>(), new Resource(6, ByteAmount.ZERO, ByteAmount.ZERO)));
    when(mockScheduler.addContainers(any())).thenReturn(mockRetrunSet);
    UpdateTopologyManager spyUpdateManager = spyUpdateManager(mockStateMgr, mockScheduler, testTopology);
    PowerMockito.spy(TManagerUtils.class);
    PowerMockito.doNothing().when(TManagerUtils.class, "sendToTManager", any(String.class), eq(TOPOLOGY_NAME), eq(mockStateMgr), any(NetworkUtils.TunnelConfig.class));
    // reactivation won't happen since topology state is still running due to mock state manager
    PowerMockito.doNothing().when(TManagerUtils.class, "transitionTopologyState", eq(TOPOLOGY_NAME), eq(TManagerUtils.TManagerCommand.ACTIVATE), eq(mockStateMgr), eq(TopologyAPI.TopologyState.PAUSED), eq(TopologyAPI.TopologyState.RUNNING), any(NetworkUtils.TunnelConfig.class));
    spyUpdateManager.updateTopology(currentProtoPlan, proposedProtoPlan);
    verify(spyUpdateManager).deactivateTopology(eq(mockStateMgr), eq(testTopology), eq(proposedPacking));
    verify(spyUpdateManager).reactivateTopology(eq(mockStateMgr), eq(testTopology), eq(2));
    verify(mockScheduler).addContainers(expectedContainersToAdd);
    verify(mockScheduler).removeContainers(expectedContainersToRemove);
    verify(lock).tryLock(any(Long.class), any(TimeUnit.class));
    verify(lock).unlock();
    PowerMockito.verifyStatic(times(1));
    TManagerUtils.transitionTopologyState(eq(TOPOLOGY_NAME), eq(TManagerUtils.TManagerCommand.DEACTIVATE), eq(mockStateMgr), eq(TopologyAPI.TopologyState.RUNNING), eq(TopologyAPI.TopologyState.PAUSED), any(NetworkUtils.TunnelConfig.class));
    PowerMockito.verifyStatic(times(1));
    TManagerUtils.transitionTopologyState(eq(TOPOLOGY_NAME), eq(TManagerUtils.TManagerCommand.ACTIVATE), eq(mockStateMgr), eq(TopologyAPI.TopologyState.PAUSED), eq(TopologyAPI.TopologyState.RUNNING), any(NetworkUtils.TunnelConfig.class));
}

Example 43 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class V1ControllerTest method testConfigureContainerResources.

@Test
public void testConfigureContainerResources() {
    final boolean isExecutor = true;
    final Resource resourceDefault = new Resource(9, ByteAmount.fromMegabytes(19000), ByteAmount.fromMegabytes(99000));
    final Resource resourceCustom = new Resource(4, ByteAmount.fromMegabytes(34000), ByteAmount.fromMegabytes(400000));
    final Quantity defaultRAM = Quantity.fromString(KubernetesUtils.Megabytes(resourceDefault.getRam()));
    final Quantity defaultCPU = Quantity.fromString(Double.toString(KubernetesUtils.roundDecimal(resourceDefault.getCpu(), 3)));
    final Quantity customRAM = Quantity.fromString(KubernetesUtils.Megabytes(resourceCustom.getRam()));
    final Quantity customCPU = Quantity.fromString(Double.toString(KubernetesUtils.roundDecimal(resourceCustom.getCpu(), 3)));
    final Quantity customDisk = Quantity.fromString(KubernetesUtils.Megabytes(resourceCustom.getDisk()));
    final Config configNoLimit = Config.newBuilder().put(KubernetesContext.KUBERNETES_RESOURCE_REQUEST_MODE, "NOT_SET").build();
    final Config configWithLimit = Config.newBuilder().put(KubernetesContext.KUBERNETES_RESOURCE_REQUEST_MODE, "EQUAL_TO_LIMIT").build();
    final V1ResourceRequirements expectDefaultRequirements = new V1ResourceRequirements().putLimitsItem(KubernetesConstants.MEMORY, defaultRAM).putLimitsItem(KubernetesConstants.CPU, defaultCPU);
    final V1ResourceRequirements expectCustomRequirements = new V1ResourceRequirements().putLimitsItem(KubernetesConstants.MEMORY, defaultRAM).putLimitsItem(KubernetesConstants.CPU, defaultCPU).putLimitsItem("disk", customDisk);
    final V1ResourceRequirements customRequirements = new V1ResourceRequirements().putLimitsItem(KubernetesConstants.MEMORY, customRAM).putLimitsItem(KubernetesConstants.CPU, customCPU).putLimitsItem("disk", customDisk);
    // Default. Null resources.
    V1Container containerNull = new V1ContainerBuilder().build();
    v1ControllerWithPodTemplate.configureContainerResources(containerNull, configNoLimit, resourceDefault, isExecutor);
    Assert.assertTrue("Default LIMITS should be set in container with null LIMITS", containerNull.getResources().getLimits().entrySet().containsAll(expectDefaultRequirements.getLimits().entrySet()));
    // Empty resources.
    V1Container containerEmpty = new V1ContainerBuilder().withNewResources().endResources().build();
    v1ControllerWithPodTemplate.configureContainerResources(containerEmpty, configNoLimit, resourceDefault, isExecutor);
    Assert.assertTrue("Default LIMITS should be set in container with empty LIMITS", containerNull.getResources().getLimits().entrySet().containsAll(expectDefaultRequirements.getLimits().entrySet()));
    // Custom resources.
    V1Container containerCustom = new V1ContainerBuilder().withResources(customRequirements).build();
    v1ControllerWithPodTemplate.configureContainerResources(containerCustom, configNoLimit, resourceDefault, isExecutor);
    Assert.assertTrue("Custom LIMITS should be set in container with custom LIMITS", containerCustom.getResources().getLimits().entrySet().containsAll(expectCustomRequirements.getLimits().entrySet()));
    // Custom resources with request.
    V1Container containerRequests = new V1ContainerBuilder().withResources(customRequirements).build();
    v1ControllerWithPodTemplate.configureContainerResources(containerRequests, configWithLimit, resourceDefault, isExecutor);
    Assert.assertTrue("Custom LIMITS should be set in container with custom LIMITS and REQUEST", containerRequests.getResources().getLimits().entrySet().containsAll(expectCustomRequirements.getLimits().entrySet()));
    Assert.assertTrue("Custom REQUEST should be set in container with custom LIMITS and REQUEST", containerRequests.getResources().getRequests().entrySet().containsAll(expectCustomRequirements.getLimits().entrySet()));
}

Example 44 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class MesosSchedulerTest method testGetBaseContainer.

@Test
public void testGetBaseContainer() throws Exception {
    final double CPU = 0.5;
    final ByteAmount MEM = ByteAmount.fromMegabytes(100);
    final ByteAmount DISK = ByteAmount.fromMegabytes(100);
    Resource containerResources = new Resource(CPU, MEM, DISK);
    PackingPlan.ContainerPlan containerPlan = new PackingPlan.ContainerPlan(0, new HashSet<PackingPlan.InstancePlan>(), containerResources);
    Set<PackingPlan.ContainerPlan> containerPlans = new HashSet<>();
    containerPlans.add(containerPlan);
    PackingPlan packingPlan = new PackingPlan(TOPOLOGY_NAME, containerPlans);
    BaseContainer container = scheduler.getBaseContainer(0, packingPlan);
    // Assert we have constructed the correct BaseContainer structure
    Assert.assertEquals(ROLE, container.runAsUser);
    Assert.assertEquals(CPU, container.cpu, 0.01);
    Assert.assertEquals(MEM, ByteAmount.fromMegabytes(((Double) container.memInMB).longValue()));
    Assert.assertEquals(DISK, ByteAmount.fromMegabytes(((Double) container.diskInMB).longValue()));
    Assert.assertEquals(SchedulerUtils.ExecutorPort.getRequiredPorts().size(), container.ports);
    Assert.assertEquals(2, container.dependencies.size());
    Assert.assertTrue(container.dependencies.contains(CORE_PACKAGE_URI));
    Assert.assertTrue(container.dependencies.contains(TOPOLOGY_PACKAGE_URI));
    Assert.assertTrue(container.environmentVariables.isEmpty());
    Assert.assertTrue(container.name.startsWith("container_0"));
    // Convert to JSON
    String str = container.toString();
    String json = BaseContainer.getJobDefinitionInJSON(container);
    Assert.assertEquals(json, str);
    // Convert the JSON back to BaseContainer
    BaseContainer newContainer = BaseContainer.getJobFromJSONString(json);
    Assert.assertEquals(json, BaseContainer.getJobDefinitionInJSON(newContainer));
}

Example 45 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class V1ControllerTest method testConfigureContainerResourcesCLI.

@Test
public void testConfigureContainerResourcesCLI() {
    final boolean isExecutor = true;
    final String customLimitMEMStr = "120Gi";
    final String customLimitCPUStr = "5";
    final String customRequestMEMStr = "100Mi";
    final String customRequestCPUStr = "4";
    final Resource resources = new Resource(6, ByteAmount.fromMegabytes(34000), ByteAmount.fromGigabytes(400));
    final Quantity customLimitMEM = Quantity.fromString(customLimitMEMStr);
    final Quantity customLimitCPU = Quantity.fromString(customLimitCPUStr);
    final Quantity customRequestMEM = Quantity.fromString(customRequestMEMStr);
    final Quantity customRequestCPU = Quantity.fromString(customRequestCPUStr);
    final Config config = Config.newBuilder().put(String.format(KubernetesContext.KUBERNETES_RESOURCE_LIMITS_PREFIX + KubernetesConstants.CPU, KubernetesConstants.EXECUTOR_NAME), customLimitCPUStr).put(String.format(KubernetesContext.KUBERNETES_RESOURCE_LIMITS_PREFIX + KubernetesConstants.MEMORY, KubernetesConstants.EXECUTOR_NAME), customLimitMEMStr).put(String.format(KubernetesContext.KUBERNETES_RESOURCE_REQUESTS_PREFIX + KubernetesConstants.CPU, KubernetesConstants.EXECUTOR_NAME), customRequestCPUStr).put(String.format(KubernetesContext.KUBERNETES_RESOURCE_REQUESTS_PREFIX + KubernetesConstants.MEMORY, KubernetesConstants.EXECUTOR_NAME), customRequestMEMStr).put(KubernetesContext.KUBERNETES_RESOURCE_REQUEST_MODE, "EQUAL_TO_LIMIT").build();
    final V1Container expected = new V1ContainerBuilder().withNewResources().addToLimits(KubernetesConstants.CPU, customLimitCPU).addToLimits(KubernetesConstants.MEMORY, customLimitMEM).addToRequests(KubernetesConstants.CPU, customRequestCPU).addToRequests(KubernetesConstants.MEMORY, customRequestMEM).endResources().build();
    final V1Container actual = new V1Container();
    v1ControllerWithPodTemplate.configureContainerResources(actual, config, resources, isExecutor);
    Assert.assertEquals("Container Resources are set from CLI.", expected, actual);
}