Search in sources :

Example 1 with SchedulerUtils

use of org.apache.heron.scheduler.utils.SchedulerUtils in project heron by twitter.

the class V1Controller method getExecutorCommand.

/**
 * Generates the command to start Heron within the <code>container</code>.
 * @param containerId Passed down to <>SchedulerUtils</> to generate executor command.
 * @param numOfInstances Used to configure the debugging ports.
 * @param isExecutor Flag used to generate the correct <code>shard_id</code>.
 * @return The complete command to start Heron in a <code>container</code>.
 */
protected List<String> getExecutorCommand(String containerId, int numOfInstances, boolean isExecutor) {
    final Config configuration = getConfiguration();
    final Config runtimeConfiguration = getRuntimeConfiguration();
    final Map<ExecutorPort, String> ports = KubernetesConstants.EXECUTOR_PORTS.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString()));
    if (TopologyUtils.getTopologyRemoteDebuggingEnabled(Runtime.topology(runtimeConfiguration)) && numOfInstances != 0) {
        List<String> remoteDebuggingPorts = new LinkedList<>();
        IntStream.range(0, numOfInstances).forEach(i -> {
            int port = KubernetesConstants.JVM_REMOTE_DEBUGGER_PORT + i;
            remoteDebuggingPorts.add(String.valueOf(port));
        });
        ports.put(ExecutorPort.JVM_REMOTE_DEBUGGER_PORTS, String.join(",", remoteDebuggingPorts));
    }
    final String[] executorCommand = SchedulerUtils.getExecutorCommand(configuration, runtimeConfiguration, containerId, ports);
    return Arrays.asList("sh", "-c", KubernetesUtils.getConfCommand(configuration) + " && " + KubernetesUtils.getFetchCommand(configuration, runtimeConfiguration) + " && " + setShardIdEnvironmentVariableCommand(isExecutor) + " && " + String.join(" ", executorCommand));
}
Also used : Arrays(java.util.Arrays) V1Toleration(io.kubernetes.client.openapi.models.V1Toleration) V1ResourceRequirements(io.kubernetes.client.openapi.models.V1ResourceRequirements) V1VolumeBuilder(io.kubernetes.client.openapi.models.V1VolumeBuilder) PackingPlan(org.apache.heron.spi.packing.PackingPlan) V1Patch(io.kubernetes.client.custom.V1Patch) Configuration(io.kubernetes.client.openapi.Configuration) V1EnvVar(io.kubernetes.client.openapi.models.V1EnvVar) Map(java.util.Map) V1StatefulSet(io.kubernetes.client.openapi.models.V1StatefulSet) Quantity(io.kubernetes.client.custom.Quantity) V1ObjectFieldSelector(io.kubernetes.client.openapi.models.V1ObjectFieldSelector) V1VolumeMount(io.kubernetes.client.openapi.models.V1VolumeMount) V1PersistentVolumeClaimBuilder(io.kubernetes.client.openapi.models.V1PersistentVolumeClaimBuilder) V1Volume(io.kubernetes.client.openapi.models.V1Volume) Pair(org.apache.heron.common.basics.Pair) TopologyRuntimeManagementException(org.apache.heron.scheduler.TopologyRuntimeManagementException) SchedulerUtils(org.apache.heron.scheduler.utils.SchedulerUtils) Set(java.util.Set) V1LabelSelector(io.kubernetes.client.openapi.models.V1LabelSelector) Logger(java.util.logging.Logger) Runtime(org.apache.heron.scheduler.utils.Runtime) V1Status(io.kubernetes.client.openapi.models.V1Status) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) List(java.util.List) V1PodTemplate(io.kubernetes.client.openapi.models.V1PodTemplate) V1PersistentVolumeClaim(io.kubernetes.client.openapi.models.V1PersistentVolumeClaim) V1SecretKeySelector(io.kubernetes.client.openapi.models.V1SecretKeySelector) V1ServiceSpec(io.kubernetes.client.openapi.models.V1ServiceSpec) V1EnvVarSource(io.kubernetes.client.openapi.models.V1EnvVarSource) ExecutorPort(org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort) PatchUtils(io.kubernetes.client.util.PatchUtils) HTTP_NOT_FOUND(java.net.HttpURLConnection.HTTP_NOT_FOUND) IntStream(java.util.stream.IntStream) HashMap(java.util.HashMap) V1PodTemplateSpec(io.kubernetes.client.openapi.models.V1PodTemplateSpec) AppsV1Api(io.kubernetes.client.openapi.apis.AppsV1Api) ArrayList(java.util.ArrayList) Level(java.util.logging.Level) ApiClient(io.kubernetes.client.openapi.ApiClient) ApiException(io.kubernetes.client.openapi.ApiException) V1VolumeMountBuilder(io.kubernetes.client.openapi.models.V1VolumeMountBuilder) V1StatefulSetSpec(io.kubernetes.client.openapi.models.V1StatefulSetSpec) V1ConfigMap(io.kubernetes.client.openapi.models.V1ConfigMap) V1ObjectMeta(io.kubernetes.client.openapi.models.V1ObjectMeta) Resource(org.apache.heron.spi.packing.Resource) Response(okhttp3.Response) V1ContainerPort(io.kubernetes.client.openapi.models.V1ContainerPort) V1SecretVolumeSourceBuilder(io.kubernetes.client.openapi.models.V1SecretVolumeSourceBuilder) LinkedList(java.util.LinkedList) V1Container(io.kubernetes.client.openapi.models.V1Container) IOException(java.io.IOException) CoreV1Api(io.kubernetes.client.openapi.apis.CoreV1Api) Yaml(io.kubernetes.client.util.Yaml) Config(org.apache.heron.spi.common.Config) V1PodSpec(io.kubernetes.client.openapi.models.V1PodSpec) TopologyUtils(org.apache.heron.api.utils.TopologyUtils) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Comparator(java.util.Comparator) Collections(java.util.Collections) TopologySubmissionException(org.apache.heron.scheduler.TopologySubmissionException) V1Service(io.kubernetes.client.openapi.models.V1Service) Config(org.apache.heron.spi.common.Config) ExecutorPort(org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort) Map(java.util.Map) HashMap(java.util.HashMap) V1ConfigMap(io.kubernetes.client.openapi.models.V1ConfigMap) LinkedList(java.util.LinkedList)

Aggregations

VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 Quantity (io.kubernetes.client.custom.Quantity)1 V1Patch (io.kubernetes.client.custom.V1Patch)1 ApiClient (io.kubernetes.client.openapi.ApiClient)1 ApiException (io.kubernetes.client.openapi.ApiException)1 Configuration (io.kubernetes.client.openapi.Configuration)1 AppsV1Api (io.kubernetes.client.openapi.apis.AppsV1Api)1 CoreV1Api (io.kubernetes.client.openapi.apis.CoreV1Api)1 V1ConfigMap (io.kubernetes.client.openapi.models.V1ConfigMap)1 V1Container (io.kubernetes.client.openapi.models.V1Container)1 V1ContainerPort (io.kubernetes.client.openapi.models.V1ContainerPort)1 V1EnvVar (io.kubernetes.client.openapi.models.V1EnvVar)1 V1EnvVarSource (io.kubernetes.client.openapi.models.V1EnvVarSource)1 V1LabelSelector (io.kubernetes.client.openapi.models.V1LabelSelector)1 V1ObjectFieldSelector (io.kubernetes.client.openapi.models.V1ObjectFieldSelector)1 V1ObjectMeta (io.kubernetes.client.openapi.models.V1ObjectMeta)1 V1PersistentVolumeClaim (io.kubernetes.client.openapi.models.V1PersistentVolumeClaim)1 V1PersistentVolumeClaimBuilder (io.kubernetes.client.openapi.models.V1PersistentVolumeClaimBuilder)1 V1PodSpec (io.kubernetes.client.openapi.models.V1PodSpec)1 V1PodTemplate (io.kubernetes.client.openapi.models.V1PodTemplate)1