Examples with Config com.twitter.heron.spi.common.Config used on opensource projects

Example 61 with Config

use of com.twitter.heron.spi.common.Config in project incubator-heron by apache.

the class SchedulerMain method createInstance.

public static SchedulerMain createInstance(String cluster, String role, String env, String topologyJar, String topologyName, int httpPort, Boolean verbose, Properties schedulerProperties) throws IOException, InvalidTopologyException {
    // Look up the topology def file location
    String topologyDefnFile = TopologyUtils.lookUpTopologyDefnFile(".", topologyName);
    // load the topology definition into topology proto
    TopologyAPI.Topology topology = TopologyUtils.getTopology(topologyDefnFile);
    // build the config by expanding all the variables
    Config schedulerConfig = SchedulerConfigUtils.loadConfig(cluster, role, env, topologyJar, topologyDefnFile, verbose, topology);
    // set up logging with complete Config
    setupLogging(schedulerConfig);
    // Create a new instance
    SchedulerMain schedulerMain = new SchedulerMain(schedulerConfig, topology, httpPort, schedulerProperties);
    LOG.log(Level.INFO, "Loaded scheduler config: {0}", schedulerMain.config);
    return schedulerMain;
}

Example 62 with Config

use of com.twitter.heron.spi.common.Config in project incubator-heron by apache.

the class AppsV1beta1Controller method getExecutorCommand.

protected List<String> getExecutorCommand(String containerId) {
    final Map<ExecutorPort, String> ports = KubernetesConstants.EXECUTOR_PORTS.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString()));
    final Config configuration = getConfiguration();
    final Config runtimeConfiguration = getRuntimeConfiguration();
    final String[] executorCommand = SchedulerUtils.getExecutorCommand(configuration, runtimeConfiguration, containerId, ports);
    return Arrays.asList("sh", "-c", KubernetesUtils.getFetchCommand(configuration, runtimeConfiguration) + " && " + setShardIdEnvironmentVariableCommand() + " && " + String.join(" ", executorCommand));
}

Example 63 with Config

use of com.twitter.heron.spi.common.Config in project incubator-heron by apache.

the class AppsV1beta1Controller method createStatefulSet.

private V1beta1StatefulSet createStatefulSet(Resource containerResource, int numberOfInstances) {
    final String topologyName = getTopologyName();
    final Config runtimeConfiguration = getRuntimeConfiguration();
    final V1beta1StatefulSet statefulSet = new V1beta1StatefulSet();
    // setup stateful set metadata
    final V1ObjectMeta objectMeta = new V1ObjectMeta();
    objectMeta.name(topologyName);
    statefulSet.metadata(objectMeta);
    // create the stateful set spec
    final V1beta1StatefulSetSpec statefulSetSpec = new V1beta1StatefulSetSpec();
    statefulSetSpec.serviceName(topologyName);
    statefulSetSpec.setReplicas(Runtime.numContainers(runtimeConfiguration).intValue());
    // Parallel pod management tells the StatefulSet controller to launch or terminate
    // all Pods in parallel, and not to wait for Pods to become Running and Ready or completely
    // terminated prior to launching or terminating another Pod.
    statefulSetSpec.setPodManagementPolicy("Parallel");
    // add selector match labels "app=heron" and "topology=topology-name"
    // so the we know which pods to manage
    final V1LabelSelector selector = new V1LabelSelector();
    selector.matchLabels(getMatchLabels(topologyName));
    statefulSetSpec.selector(selector);
    // create a pod template
    final V1PodTemplateSpec podTemplateSpec = new V1PodTemplateSpec();
    // set up pod meta
    final V1ObjectMeta templateMetaData = new V1ObjectMeta().labels(getLabels(topologyName));
    templateMetaData.annotations(getPrometheusAnnotations());
    podTemplateSpec.setMetadata(templateMetaData);
    final List<String> command = getExecutorCommand("$" + ENV_SHARD_ID);
    podTemplateSpec.spec(getPodSpec(command, containerResource, numberOfInstances));
    statefulSetSpec.setTemplate(podTemplateSpec);
    statefulSet.spec(statefulSetSpec);
    return statefulSet;
}

Example 64 with Config

use of com.twitter.heron.spi.common.Config in project incubator-heron by apache.

the class AppsV1beta1Controller method addVolumesIfPresent.

private void addVolumesIfPresent(V1PodSpec spec) {
    final Config config = getConfiguration();
    if (KubernetesContext.hasVolume(config)) {
        final V1Volume volume = Volumes.get().create(config);
        if (volume != null) {
            LOG.fine("Adding volume: " + volume.toString());
            spec.volumes(Collections.singletonList(volume));
        }
    }
}

Example 65 with Config

use of com.twitter.heron.spi.common.Config in project incubator-heron by apache.

the class LauncherUtilsTest method constructsRuntimeWithPackingProperly.

@Test
public void constructsRuntimeWithPackingProperly() {
    Config runtime = Config.newBuilder().put("key-23", "value-34").build();
    Assert.assertNull(Runtime.componentRamMap(runtime));
    Set<PackingPlan.ContainerPlan> containers = new HashSet<>();
    containers.add(Mockito.mock(PackingPlan.ContainerPlan.class));
    containers.add(Mockito.mock(PackingPlan.ContainerPlan.class));
    PackingPlan mockPacking = Mockito.mock(PackingPlan.class);
    Mockito.when(mockPacking.getComponentRamDistribution()).thenReturn("ramMap");
    Mockito.when(mockPacking.getContainers()).thenReturn(containers);
    Config newRuntime = LauncherUtils.getInstance().createConfigWithPackingDetails(runtime, mockPacking);
    Assert.assertNull(Runtime.componentRamMap(runtime));
    Assert.assertEquals("ramMap", Runtime.componentRamMap(newRuntime));
    Assert.assertEquals(3, Runtime.numContainers(newRuntime).longValue());
    Assert.assertEquals("value-34", newRuntime.getStringValue("key-23"));
}