Examples with Resource com.twitter.heron.spi.packing.Resource used on opensource projects

Example 11 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class HeronMasterDriver method scheduleHeronWorkers.

/**
   * Container allocation is asynchronous. Requests all containers in the input set serially
   * to ensure allocated resources match the required resources.
   */
void scheduleHeronWorkers(Set<ContainerPlan> containers) throws ContainerAllocationException {
    for (ContainerPlan containerPlan : containers) {
        int id = containerPlan.getId();
        if (containerPlans.containsKey(containerPlan.getId())) {
            throw new ContainerAllocationException("Received duplicate allocation request for " + id);
        }
        Resource reqResource = containerPlan.getRequiredResource();
        containerPlans.put(id, containerPlan);
        requestContainerForWorker(id, new HeronWorker(id, reqResource));
    }
}

Example 12 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class PackingUtilsTest method testResourceScaleUp.

@Test
public void testResourceScaleUp() {
    int noSpouts = 6;
    int noBolts = 3;
    int boltScalingUp = 2;
    com.twitter.heron.api.Config topologyConfig = new com.twitter.heron.api.Config();
    TopologyAPI.Topology topology = getTopology(noSpouts, noBolts, topologyConfig);
    Config config = PackingTestUtils.newTestConfig(topology);
    Resource defaultInstanceResources = new Resource(Context.instanceCpu(config), Context.instanceRam(config), Context.instanceDisk(config));
    Map<String, Integer> componentChanges = new HashMap<>();
    // 5 bolts
    componentChanges.put("bolt", boltScalingUp);
    Resource scaleupResource = PackingUtils.computeTotalResourceChange(topology, componentChanges, defaultInstanceResources, PackingUtils.ScalingDirection.UP);
    Assert.assertEquals((long) (boltScalingUp * defaultInstanceResources.getCpu()), (long) scaleupResource.getCpu());
    Assert.assertEquals(defaultInstanceResources.getRam().multiply(boltScalingUp), scaleupResource.getRam());
    Assert.assertEquals(defaultInstanceResources.getDisk().multiply(boltScalingUp), scaleupResource.getDisk());
}

Example 13 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class PackingUtilsTest method testResourceScaleDown.

@Test
public void testResourceScaleDown() {
    int noSpouts = 6;
    int noBolts = 3;
    int boltScalingDown = 2;
    com.twitter.heron.api.Config topologyConfig = new com.twitter.heron.api.Config();
    TopologyAPI.Topology topology = getTopology(noSpouts, noBolts, topologyConfig);
    Config config = PackingTestUtils.newTestConfig(topology);
    Resource defaultInstanceResources = new Resource(Context.instanceCpu(config), Context.instanceRam(config), Context.instanceDisk(config));
    Map<String, Integer> componentChanges = new HashMap<>();
    // 1 bolt
    componentChanges.put("bolt", -boltScalingDown);
    Resource scaledownResource = PackingUtils.computeTotalResourceChange(topology, componentChanges, defaultInstanceResources, PackingUtils.ScalingDirection.DOWN);
    Assert.assertEquals((long) (boltScalingDown * defaultInstanceResources.getCpu()), (long) scaledownResource.getCpu());
    Assert.assertEquals(defaultInstanceResources.getRam().multiply(boltScalingDown), scaledownResource.getRam());
    Assert.assertEquals(defaultInstanceResources.getDisk().multiply(boltScalingDown), scaledownResource.getDisk());
}

Example 14 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class AuroraSchedulerTest method testProperties.

@Test
public void testProperties() throws URISyntaxException {
    TopologyAPI.Topology topology = TopologyTests.createTopology(TOPOLOGY_NAME, new com.twitter.heron.api.Config(), "spoutName", "boltName", 1, 1);
    Config runtime = mock(Config.class);
    when(runtime.get(Key.TOPOLOGY_DEFINITION)).thenReturn(topology);
    when(runtime.get(Key.TOPOLOGY_PACKAGE_URI)).thenReturn(new URI("http://foo/bar"));
    // This must mimic how SubmitterMain loads configs
    CommandLine commandLine = mock(CommandLine.class);
    when(commandLine.getOptionValue("cluster")).thenReturn("some_cluster");
    when(commandLine.getOptionValue("role")).thenReturn("some_role");
    when(commandLine.getOptionValue("environment")).thenReturn("some_env");
    when(commandLine.getOptionValue("heron_home")).thenReturn("/some/heron/home");
    when(commandLine.getOptionValue("config_path")).thenReturn("/some/config/path");
    when(commandLine.getOptionValue("topology_package")).thenReturn("jar");
    when(commandLine.getOptionValue("topology_defn")).thenReturn("/mock/defnFile.defn");
    when(commandLine.getOptionValue("topology_bin")).thenReturn("/mock/binaryFile.jar");
    Config config = Mockito.spy(SubmitterMain.loadConfig(commandLine, topology));
    AuroraScheduler testScheduler = new AuroraScheduler();
    testScheduler.initialize(config, runtime);
    Resource containerResource = new Resource(2.3, ByteAmount.fromGigabytes(2), ByteAmount.fromGigabytes(3));
    Map<AuroraField, String> properties = testScheduler.createAuroraProperties(containerResource);
    // this part is key, the conf path in the config is absolute to the install dir, but what
    // aurora properties get below is the relative ./heron-conf path to be used when run remotely
    assertEquals("Invalid value for key " + Key.HERON_CONF, "/some/config/path", config.getStringValue(Key.HERON_CONF));
    String expectedConf = "./heron-conf";
    String expectedBin = "./heron-core/bin";
    String expectedLib = "./heron-core/lib";
    String expectedDist = "./heron-core/dist";
    for (AuroraField field : AuroraField.values()) {
        boolean asserted = false;
        Object expected = null;
        Object found = properties.get(field);
        switch(field) {
            case CLUSTER:
                expected = "some_cluster";
                break;
            case ENVIRON:
                expected = "some_env";
                break;
            case ROLE:
                expected = "some_role";
                break;
            case COMPONENT_RAMMAP:
            case STATEMGR_CONNECTION_STRING:
            case STATEMGR_ROOT_PATH:
                expected = null;
                break;
            case COMPONENT_JVM_OPTS_IN_BASE64:
            case INSTANCE_JVM_OPTS_IN_BASE64:
                expected = "\"\"";
                break;
            case CORE_PACKAGE_URI:
                expected = expectedDist + "/heron-core.tar.gz";
                break;
            case CPUS_PER_CONTAINER:
                expected = Double.valueOf(containerResource.getCpu()).toString();
                break;
            case DISK_PER_CONTAINER:
                expected = Long.valueOf(containerResource.getDisk().asBytes()).toString();
                break;
            case RAM_PER_CONTAINER:
                expected = Long.valueOf(containerResource.getRam().asBytes()).toString();
                break;
            case JAVA_HOME:
                expected = "/usr/lib/jvm/default-java";
                break;
            case IS_PRODUCTION:
                expected = Boolean.FALSE.toString();
                break;
            case NUM_CONTAINERS:
                expected = "2";
                break;
            case EXECUTOR_BINARY:
                expected = expectedBin + "/heron-executor";
                break;
            case INSTANCE_CLASSPATH:
                expected = expectedLib + "/instance/*";
                break;
            case METRICSMGR_CLASSPATH:
                expected = expectedLib + "/metricsmgr/*";
                break;
            case METRICS_YAML:
                expected = expectedConf + "/metrics_sinks.yaml";
                break;
            case PYTHON_INSTANCE_BINARY:
                expected = expectedBin + "/heron-python-instance";
                break;
            case SCHEDULER_CLASSPATH:
                expected = expectedLib + "/scheduler/*:./heron-core/lib/packing/*:./heron-core/lib/statemgr/*";
                break;
            case SHELL_BINARY:
                expected = expectedBin + "/heron-shell";
                break;
            case STMGR_BINARY:
                expected = expectedBin + "/heron-stmgr";
                break;
            case TMASTER_BINARY:
                expected = expectedBin + "/heron-tmaster";
                break;
            case SYSTEM_YAML:
                expected = expectedConf + "/heron_internals.yaml";
                break;
            case TOPOLOGY_BINARY_FILE:
            case TOPOLOGY_CLASSPATH:
                expected = "binaryFile.jar";
                break;
            case TOPOLOGY_DEFINITION_FILE:
                expected = "defnFile.defn";
                break;
            case TOPOLOGY_ID:
                assertTrue(field + " does not start with topologyName: " + found, found.toString().startsWith("topologyName"));
                asserted = true;
                break;
            case TOPOLOGY_NAME:
                expected = "topologyName";
                break;
            case TOPOLOGY_PACKAGE_TYPE:
                expected = "jar";
                break;
            case TOPOLOGY_PACKAGE_URI:
                expected = "http://foo/bar";
                break;
            case METRICSCACHEMGR_CLASSPATH:
                expected = expectedLib + "/metricscachemgr/*";
                break;
            default:
                fail(String.format("Expected value for Aurora field %s not found in test (found=%s)", field, found));
        }
        if (!asserted) {
            assertEquals("Incorrect value found for field " + field, expected, found);
        }
        properties.remove(field);
    }
    assertTrue("The following aurora fields were not set by the scheduler: " + properties, properties.isEmpty());
}

Example 15 with Resource

use of com.twitter.heron.spi.packing.Resource in project heron by twitter.

the class MarathonScheduler method getTopologyConf.

protected String getTopologyConf(PackingPlan packing) {
    ObjectMapper mapper = new ObjectMapper();
    // TODO (nlu): use heterogeneous resources
    // Align resources to maximal requested resource
    PackingPlan updatedPackingPlan = packing.cloneWithHomogeneousScheduledResource();
    SchedulerUtils.persistUpdatedPackingPlan(Runtime.topologyName(runtime), updatedPackingPlan, Runtime.schedulerStateManagerAdaptor(runtime));
    Resource containerResource = updatedPackingPlan.getContainers().iterator().next().getRequiredResource();
    // Create app conf list for each container
    ArrayNode instances = mapper.createArrayNode();
    for (int i = 0; i < Runtime.numContainers(runtime); i++) {
        ObjectNode instance = mapper.createObjectNode();
        instance.put(MarathonConstants.ID, Integer.toString(i));
        instance.put(MarathonConstants.COMMAND, getExecutorCommand(i));
        instance.put(MarathonConstants.CPU, containerResource.getCpu());
        instance.put(MarathonConstants.MEMORY, containerResource.getRam().asMegabytes());
        instance.put(MarathonConstants.DISK, containerResource.getDisk().asMegabytes());
        instance.set(MarathonConstants.PORT_DEFINITIONS, getPorts(mapper));
        instance.put(MarathonConstants.INSTANCES, 1);
        instance.set(MarathonConstants.LABELS, getLabels(mapper));
        instance.set(MarathonConstants.FETCH, getFetchList(mapper));
        instance.put(MarathonConstants.USER, Context.role(config));
        instances.add(instance);
    }
    // Create marathon group for a topology
    ObjectNode appConf = mapper.createObjectNode();
    appConf.put(MarathonConstants.ID, Runtime.topologyName(runtime));
    appConf.set(MarathonConstants.APPS, instances);
    return appConf.toString();
}