Examples with Resource org.apache.heron.spi.packing.Resource used on opensource projects

Example 16 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class PackingTestUtils method testContainerPlan.

@SafeVarargs
public static PackingPlan.ContainerPlan testContainerPlan(int containerId, Pair<String, Integer>... instanceInfo) {
    double cpu = 1.5;
    ByteAmount ram = ByteAmount.fromGigabytes(1);
    Set<PackingPlan.InstancePlan> instancePlans = new HashSet<>();
    for (Pair<String, Integer> info : instanceInfo) {
        PackingPlan.InstancePlan instance = testInstancePlan(info.first, info.second);
        instancePlans.add(instance);
        cpu += instance.getResource().getCpu();
        ram = ram.plus(instance.getResource().getRam());
    }
    Resource resource = new Resource(cpu, ram, ram);
    return new PackingPlan.ContainerPlan(containerId, instancePlans, resource);
}

Example 17 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class NomadSchedulerTest method testGetTaskDocker.

@SuppressWarnings("unchecked")
@Test
public void testGetTaskDocker() {
    this.mockRuntime = this.mockRuntime.newBuilder().putAll(this.mockRuntime).put(NomadContext.HERON_NOMAD_DRIVER, NomadConstants.NomadDriver.DOCKER.getName()).build();
    this.mockConfig = this.mockConfig.newBuilder().putAll(this.mockConfig).put(NomadContext.HERON_NOMAD_DRIVER, NomadConstants.NomadDriver.DOCKER.getName()).build();
    Set<PackingPlan.ContainerPlan> containers = new HashSet<>();
    containers.add(Mockito.mock(PackingPlan.ContainerPlan.class));
    PowerMockito.mockStatic(SchedulerUtils.class);
    Resource resource = new Resource(CPU_RESOURCE, MEMORY_RESOURCE, DISK_RESOURCE);
    PowerMockito.when(SchedulerUtils.executorCommandArgs(Mockito.any(), Mockito.any(), Mockito.anyMap(), Mockito.anyString())).thenReturn(EXECUTOR_CMD_ARGS);
    PowerMockito.mockStatic(NomadScheduler.class);
    PowerMockito.when(NomadScheduler.getFetchCommand(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(TOPOLOGY_DOWNLOAD_CMD);
    PowerMockito.when(NomadScheduler.getHeronNomadScript(this.mockConfig)).thenReturn(HERON_NOMAD_SCRIPT);
    PowerMockito.when(NomadScheduler.longToInt(MEMORY_RESOURCE.asMegabytes())).thenReturn((int) MEMORY_RESOURCE.asMegabytes());
    PowerMockito.when(NomadScheduler.longToInt(DISK_RESOURCE.asMegabytes())).thenReturn((int) DISK_RESOURCE.asMegabytes());
    PowerMockito.when(NomadScheduler.getPrometheusMetricsFile(Mockito.any())).thenReturn(PORT_FILE);
    scheduler.initialize(this.mockConfig, this.mockRuntime);
    Task task = scheduler.getTask(TASK_NAME, CONTAINER_INDEX, resource);
    LOG.info("task: " + task);
    Assert.assertEquals(TASK_NAME, task.getName());
    Assert.assertEquals(NomadConstants.NomadDriver.DOCKER.getName(), task.getDriver());
    Assert.assertTrue(task.getConfig().containsKey(NomadConstants.NOMAD_TASK_COMMAND));
    Assert.assertEquals(NomadConstants.SHELL_CMD, task.getConfig().get(NomadConstants.NOMAD_TASK_COMMAND));
    Assert.assertTrue(task.getEnv().containsKey(NomadConstants.METRICS_PORT_FILE));
    Assert.assertEquals((int) CPU_RESOURCE * HERON_NOMAD_CORE_FREQ_MAPPING, task.getResources().getCpu().intValue());
    Assert.assertEquals((int) MEMORY_RESOURCE.asMegabytes(), task.getResources().getMemoryMb().intValue());
    Assert.assertEquals((int) DISK_RESOURCE.asMegabytes(), task.getResources().getDiskMb().intValue());
    Assert.assertTrue(task.getEnv().containsKey(NomadConstants.HOST));
    Assert.assertEquals(PORT_FILE, task.getEnv().get(NomadConstants.METRICS_PORT_FILE));
    Assert.assertEquals("${attr.unique.network.ip-address}", task.getEnv().get(NomadConstants.HOST));
    Assert.assertTrue(task.getConfig().containsKey(NomadConstants.NETWORK_MODE));
    Assert.assertEquals("default", task.getConfig().get(NomadConstants.NETWORK_MODE));
    Assert.assertEquals(task.getResources().getNetworks().size(), 1);
    Set<String> ports = new HashSet<>();
    for (Port entry : task.getResources().getNetworks().get(0).getDynamicPorts()) {
        ports.add(entry.getLabel());
        Assert.assertEquals(entry.getValue(), 0);
    }
    for (SchedulerUtils.ExecutorPort entry : NomadConstants.EXECUTOR_PORTS.keySet()) {
        Assert.assertTrue(ports.contains(entry.getName().replace("-", "_")));
    }
    Assert.assertTrue(ports.contains(NomadConstants.METRICS_PORT));
}

Example 18 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class NomadSchedulerTest method testGetJob.

@Test
public void testGetJob() {
    Set<PackingPlan.ContainerPlan> containers = new HashSet<>();
    containers.add(Mockito.mock(PackingPlan.ContainerPlan.class));
    PackingPlan.ContainerPlan containerPlan = new PackingPlan.ContainerPlan(CONTAINER_INDEX, new HashSet<>(), Mockito.mock(Resource.class));
    Optional<PackingPlan.ContainerPlan> plan = Optional.of(containerPlan);
    Resource resource = new Resource(CPU_RESOURCE, MEMORY_RESOURCE, DISK_RESOURCE);
    scheduler.initialize(this.mockConfig, this.mockRuntime);
    Mockito.doReturn(new TaskGroup()).when(scheduler).getTaskGroup(Mockito.anyString(), Mockito.anyInt(), Mockito.any());
    Job job = scheduler.getJob(CONTAINER_INDEX, plan, resource);
    LOG.info("job: " + job);
    Assert.assertEquals(TOPOLOGY_ID + "-" + CONTAINER_INDEX, job.getId());
    Assert.assertEquals(TOPOLOGY_NAME + "-" + CONTAINER_INDEX, job.getName());
    Assert.assertArrayEquals(Arrays.asList(NomadConstants.NOMAD_DEFAULT_DATACENTER).toArray(), job.getDatacenters().toArray());
    Assert.assertNotNull(job.getTaskGroups());
}

Example 19 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class NomadSchedulerTest method testGetTaskRawExec.

@SuppressWarnings("unchecked")
@Test
public void testGetTaskRawExec() {
    Set<PackingPlan.ContainerPlan> containers = new HashSet<>();
    containers.add(Mockito.mock(PackingPlan.ContainerPlan.class));
    PowerMockito.mockStatic(SchedulerUtils.class);
    Resource resource = new Resource(CPU_RESOURCE, MEMORY_RESOURCE, DISK_RESOURCE);
    PowerMockito.when(SchedulerUtils.executorCommandArgs(Mockito.any(), Mockito.any(), Mockito.anyMap(), Mockito.anyString())).thenReturn(EXECUTOR_CMD_ARGS);
    PowerMockito.mockStatic(NomadScheduler.class);
    PowerMockito.when(NomadScheduler.getFetchCommand(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(TOPOLOGY_DOWNLOAD_CMD);
    PowerMockito.when(NomadScheduler.getHeronNomadScript(this.mockConfig)).thenReturn(HERON_NOMAD_SCRIPT);
    PowerMockito.when(NomadScheduler.longToInt(MEMORY_RESOURCE.asMegabytes())).thenReturn((int) MEMORY_RESOURCE.asMegabytes());
    PowerMockito.when(NomadScheduler.longToInt(DISK_RESOURCE.asMegabytes())).thenReturn((int) DISK_RESOURCE.asMegabytes());
    PowerMockito.when(NomadScheduler.getPrometheusMetricsFile(Mockito.any())).thenReturn(PORT_FILE);
    scheduler.initialize(this.mockConfig, this.mockRuntime);
    Task task = scheduler.getTask(TASK_NAME, CONTAINER_INDEX, resource);
    LOG.info("task: " + task);
    Assert.assertEquals(TASK_NAME, task.getName());
    Assert.assertEquals(NomadConstants.NomadDriver.RAW_EXEC.getName(), task.getDriver());
    Assert.assertTrue(task.getConfig().containsKey(NomadConstants.NOMAD_TASK_COMMAND));
    Assert.assertEquals(NomadConstants.SHELL_CMD, task.getConfig().get(NomadConstants.NOMAD_TASK_COMMAND));
    Assert.assertTrue(task.getConfig().containsKey(NomadConstants.NOMAD_TASK_COMMAND_ARGS));
    Assert.assertArrayEquals(Arrays.asList(NomadConstants.NOMAD_HERON_SCRIPT_NAME).toArray(), (String[]) task.getConfig().get(NomadConstants.NOMAD_TASK_COMMAND_ARGS));
    Assert.assertEquals(1, task.getTemplates().size());
    Assert.assertEquals(HERON_NOMAD_SCRIPT, task.getTemplates().get(0).getEmbeddedTmpl());
    Assert.assertEquals(NomadConstants.NOMAD_HERON_SCRIPT_NAME, task.getTemplates().get(0).getDestPath());
    Assert.assertEquals((int) CPU_RESOURCE * HERON_NOMAD_CORE_FREQ_MAPPING, task.getResources().getCpu().intValue());
    Assert.assertEquals((int) MEMORY_RESOURCE.asMegabytes(), task.getResources().getMemoryMb().intValue());
    Assert.assertEquals((int) DISK_RESOURCE.asMegabytes(), task.getResources().getDiskMb().intValue());
    Assert.assertTrue(task.getEnv().containsKey(NomadConstants.HERON_NOMAD_WORKING_DIR));
    Assert.assertTrue(task.getEnv().containsKey(NomadConstants.HERON_USE_CORE_PACKAGE_URI));
    Assert.assertTrue(task.getEnv().containsKey(NomadConstants.HERON_CORE_PACKAGE_URI));
    Assert.assertTrue(task.getEnv().containsKey(NomadConstants.HERON_TOPOLOGY_DOWNLOAD_CMD));
    Assert.assertTrue(task.getEnv().containsKey(NomadConstants.HERON_EXECUTOR_CMD));
    Assert.assertTrue(task.getEnv().containsKey(NomadConstants.METRICS_PORT_FILE));
    Assert.assertEquals(NomadKey.WORKING_DIRECTORY.getDefaultString() + "/container-" + String.valueOf(CONTAINER_INDEX), task.getEnv().get(NomadConstants.HERON_NOMAD_WORKING_DIR));
    Assert.assertEquals(USE_CORE_PACKAGE_URI.toString(), task.getEnv().get(NomadConstants.HERON_USE_CORE_PACKAGE_URI));
    Assert.assertEquals(CORE_PACKAGE_URI, task.getEnv().get(NomadConstants.HERON_CORE_PACKAGE_URI));
    Assert.assertEquals(TOPOLOGY_DOWNLOAD_CMD, task.getEnv().get(NomadConstants.HERON_TOPOLOGY_DOWNLOAD_CMD));
    Assert.assertEquals("./heron-core/bin/heron-executor args1 args2", task.getEnv().get(NomadConstants.HERON_EXECUTOR_CMD));
    Assert.assertEquals(PORT_FILE, task.getEnv().get(NomadConstants.METRICS_PORT_FILE));
}

Example 20 with Resource

use of org.apache.heron.spi.packing.Resource in project heron by twitter.

the class MesosScheduler method fillResourcesRequirementForBaseContainer.

/**
 * Fill the the resources requirement, i.e. CPU, memory and disk for the given container.
 * This method changes the BaseContainer passed in.
 * <p>
 * Notice: Currently we just make every container homogeneous,
 * requiring maximum resources for every container.
 *
 * @param container the BaseContainer to fill value in
 * @param containerIndex the index of the container
 * @param packing the packing plan
 */
protected void fillResourcesRequirementForBaseContainer(BaseContainer container, Integer containerIndex, PackingPlan packing) {
    PackingPlan updatedPackingPlan = packing.cloneWithHomogeneousScheduledResource();
    Resource maxResourceContainer = updatedPackingPlan.getContainers().iterator().next().getRequiredResource();
    double cpu = 0;
    ByteAmount disk = ByteAmount.ZERO;
    ByteAmount mem = ByteAmount.ZERO;
    for (PackingPlan.ContainerPlan cp : packing.getContainers()) {
        Resource containerResource = cp.getRequiredResource();
        cpu = Math.max(cpu, containerResource.getCpu());
        disk = disk.max(containerResource.getDisk());
        mem = mem.max(containerResource.getRam());
    }
    container.cpu = maxResourceContainer.getCpu();
    // Convert them from bytes to MB
    container.diskInMB = maxResourceContainer.getDisk().asMegabytes();
    container.memInMB = maxResourceContainer.getRam().asMegabytes();
    container.ports = SchedulerUtils.ExecutorPort.getRequiredPorts().size();
}