Search in sources :

Example 1 with ExecutorPort

use of org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort in project heron by twitter.

the class MarathonScheduler method getPorts.

protected ArrayNode getPorts(ObjectMapper mapper) {
    ArrayNode ports = mapper.createArrayNode();
    for (Map.Entry<ExecutorPort, String> entry : MarathonConstants.EXECUTOR_PORTS.entrySet()) {
        ObjectNode port = mapper.createObjectNode();
        port.put(MarathonConstants.DOCKER_CONTAINER_PORT, 0);
        port.put(MarathonConstants.PROTOCOL, MarathonConstants.TCP);
        port.put(MarathonConstants.HOST_PORT, 0);
        port.put(MarathonConstants.PORT_NAME, entry.getKey().getName());
        ports.add(port);
    }
    return ports;
}
Also used : ObjectNode(com.fasterxml.jackson.databind.node.ObjectNode) ArrayNode(com.fasterxml.jackson.databind.node.ArrayNode) ExecutorPort(org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort) Map(java.util.Map)

Example 2 with ExecutorPort

use of org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort in project heron by twitter.

the class V1Controller method getExecutorCommand.

/**
 * Generates the command to start Heron within the <code>container</code>.
 * @param containerId Passed down to <>SchedulerUtils</> to generate executor command.
 * @param numOfInstances Used to configure the debugging ports.
 * @param isExecutor Flag used to generate the correct <code>shard_id</code>.
 * @return The complete command to start Heron in a <code>container</code>.
 */
protected List<String> getExecutorCommand(String containerId, int numOfInstances, boolean isExecutor) {
    final Config configuration = getConfiguration();
    final Config runtimeConfiguration = getRuntimeConfiguration();
    final Map<ExecutorPort, String> ports = KubernetesConstants.EXECUTOR_PORTS.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString()));
    if (TopologyUtils.getTopologyRemoteDebuggingEnabled(Runtime.topology(runtimeConfiguration)) && numOfInstances != 0) {
        List<String> remoteDebuggingPorts = new LinkedList<>();
        IntStream.range(0, numOfInstances).forEach(i -> {
            int port = KubernetesConstants.JVM_REMOTE_DEBUGGER_PORT + i;
            remoteDebuggingPorts.add(String.valueOf(port));
        });
        ports.put(ExecutorPort.JVM_REMOTE_DEBUGGER_PORTS, String.join(",", remoteDebuggingPorts));
    }
    final String[] executorCommand = SchedulerUtils.getExecutorCommand(configuration, runtimeConfiguration, containerId, ports);
    return Arrays.asList("sh", "-c", KubernetesUtils.getConfCommand(configuration) + " && " + KubernetesUtils.getFetchCommand(configuration, runtimeConfiguration) + " && " + setShardIdEnvironmentVariableCommand(isExecutor) + " && " + String.join(" ", executorCommand));
}
Also used : Arrays(java.util.Arrays) V1Toleration(io.kubernetes.client.openapi.models.V1Toleration) V1ResourceRequirements(io.kubernetes.client.openapi.models.V1ResourceRequirements) V1VolumeBuilder(io.kubernetes.client.openapi.models.V1VolumeBuilder) PackingPlan(org.apache.heron.spi.packing.PackingPlan) V1Patch(io.kubernetes.client.custom.V1Patch) Configuration(io.kubernetes.client.openapi.Configuration) V1EnvVar(io.kubernetes.client.openapi.models.V1EnvVar) Map(java.util.Map) V1StatefulSet(io.kubernetes.client.openapi.models.V1StatefulSet) Quantity(io.kubernetes.client.custom.Quantity) V1ObjectFieldSelector(io.kubernetes.client.openapi.models.V1ObjectFieldSelector) V1VolumeMount(io.kubernetes.client.openapi.models.V1VolumeMount) V1PersistentVolumeClaimBuilder(io.kubernetes.client.openapi.models.V1PersistentVolumeClaimBuilder) V1Volume(io.kubernetes.client.openapi.models.V1Volume) Pair(org.apache.heron.common.basics.Pair) TopologyRuntimeManagementException(org.apache.heron.scheduler.TopologyRuntimeManagementException) SchedulerUtils(org.apache.heron.scheduler.utils.SchedulerUtils) Set(java.util.Set) V1LabelSelector(io.kubernetes.client.openapi.models.V1LabelSelector) Logger(java.util.logging.Logger) Runtime(org.apache.heron.scheduler.utils.Runtime) V1Status(io.kubernetes.client.openapi.models.V1Status) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) List(java.util.List) V1PodTemplate(io.kubernetes.client.openapi.models.V1PodTemplate) V1PersistentVolumeClaim(io.kubernetes.client.openapi.models.V1PersistentVolumeClaim) V1SecretKeySelector(io.kubernetes.client.openapi.models.V1SecretKeySelector) V1ServiceSpec(io.kubernetes.client.openapi.models.V1ServiceSpec) V1EnvVarSource(io.kubernetes.client.openapi.models.V1EnvVarSource) ExecutorPort(org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort) PatchUtils(io.kubernetes.client.util.PatchUtils) HTTP_NOT_FOUND(java.net.HttpURLConnection.HTTP_NOT_FOUND) IntStream(java.util.stream.IntStream) HashMap(java.util.HashMap) V1PodTemplateSpec(io.kubernetes.client.openapi.models.V1PodTemplateSpec) AppsV1Api(io.kubernetes.client.openapi.apis.AppsV1Api) ArrayList(java.util.ArrayList) Level(java.util.logging.Level) ApiClient(io.kubernetes.client.openapi.ApiClient) ApiException(io.kubernetes.client.openapi.ApiException) V1VolumeMountBuilder(io.kubernetes.client.openapi.models.V1VolumeMountBuilder) V1StatefulSetSpec(io.kubernetes.client.openapi.models.V1StatefulSetSpec) V1ConfigMap(io.kubernetes.client.openapi.models.V1ConfigMap) V1ObjectMeta(io.kubernetes.client.openapi.models.V1ObjectMeta) Resource(org.apache.heron.spi.packing.Resource) Response(okhttp3.Response) V1ContainerPort(io.kubernetes.client.openapi.models.V1ContainerPort) V1SecretVolumeSourceBuilder(io.kubernetes.client.openapi.models.V1SecretVolumeSourceBuilder) LinkedList(java.util.LinkedList) V1Container(io.kubernetes.client.openapi.models.V1Container) IOException(java.io.IOException) CoreV1Api(io.kubernetes.client.openapi.apis.CoreV1Api) Yaml(io.kubernetes.client.util.Yaml) Config(org.apache.heron.spi.common.Config) V1PodSpec(io.kubernetes.client.openapi.models.V1PodSpec) TopologyUtils(org.apache.heron.api.utils.TopologyUtils) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Comparator(java.util.Comparator) Collections(java.util.Collections) TopologySubmissionException(org.apache.heron.scheduler.TopologySubmissionException) V1Service(io.kubernetes.client.openapi.models.V1Service) Config(org.apache.heron.spi.common.Config) ExecutorPort(org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort) Map(java.util.Map) HashMap(java.util.HashMap) V1ConfigMap(io.kubernetes.client.openapi.models.V1ConfigMap) LinkedList(java.util.LinkedList)

Example 3 with ExecutorPort

use of org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort in project heron by twitter.

the class LocalScheduler method getExecutorCommand.

private String[] getExecutorCommand(int container, Set<PackingPlan.InstancePlan> instances) {
    Map<ExecutorPort, String> ports = new HashMap<>();
    for (ExecutorPort executorPort : ExecutorPort.getRequiredPorts()) {
        int port = SysUtils.getFreePort();
        if (port == -1) {
            throw new RuntimeException("Failed to find available ports for executor");
        }
        ports.put(executorPort, String.valueOf(port));
    }
    if (TopologyUtils.getTopologyRemoteDebuggingEnabled(Runtime.topology(runtime)) && instances != null) {
        List<String> remoteDebuggingPorts = new LinkedList<>();
        int portsForRemoteDebugging = instances.size();
        for (int i = 0; i < portsForRemoteDebugging; i++) {
            int port = SysUtils.getFreePort();
            if (port == -1) {
                throw new RuntimeException("Failed to find available ports for executor");
            }
            remoteDebuggingPorts.add(String.valueOf(port));
        }
        ports.put(ExecutorPort.JVM_REMOTE_DEBUGGER_PORTS, String.join(",", remoteDebuggingPorts));
    }
    String[] executorCmd = SchedulerUtils.getExecutorCommand(config, runtime, container, ports);
    LOG.info("Executor command line: " + Arrays.toString(executorCmd));
    return executorCmd;
}
Also used : HashMap(java.util.HashMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ExecutorPort(org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort) LinkedList(java.util.LinkedList)

Example 4 with ExecutorPort

use of org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort in project heron by twitter.

the class SlurmScheduler method getExecutorCommand.

protected String[] getExecutorCommand(PackingPlan packing) {
    Map<ExecutorPort, String> ports = new HashMap<>();
    for (ExecutorPort executorPort : ExecutorPort.getRequiredPorts()) {
        int port = SysUtils.getFreePort();
        if (port == -1) {
            throw new RuntimeException("Failed to find available ports for executor");
        }
        ports.put(executorPort, String.valueOf(port));
    }
    String[] executorCmd = SchedulerUtils.executorCommandArgs(this.config, this.runtime, ports, null);
    LOG.log(Level.FINE, "Executor command line: ", Arrays.toString(executorCmd));
    return executorCmd;
}
Also used : HashMap(java.util.HashMap) ExecutorPort(org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort)

Example 5 with ExecutorPort

use of org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort in project heron by twitter.

the class HeronExecutorTask method getExecutorCommand.

String[] getExecutorCommand() throws InvalidTopologyException {
    String topologyDefFile = getTopologyDefnFile();
    Topology topology = getTopology(topologyDefFile);
    Config config = SchedulerConfigUtils.loadConfig(cluster, role, env, topologyJar, topologyDefFile, verboseMode, topology);
    Config runtime = Config.newBuilder().put(Key.COMPONENT_RAMMAP, componentRamMap).put(Key.TOPOLOGY_DEFINITION, topology).build();
    Map<ExecutorPort, String> ports = new HashMap<>();
    for (ExecutorPort executorPort : ExecutorPort.getRequiredPorts()) {
        int port = SysUtils.getFreePort();
        if (port == -1) {
            throw new RuntimeException("Failed to find available ports for executor");
        }
        ports.put(executorPort, String.valueOf(port));
    }
    String[] executorCmd = SchedulerUtils.getExecutorCommand(config, runtime, heronExecutorId, ports);
    LOG.info("Executor command line: " + Arrays.toString(executorCmd));
    return executorCmd;
}
Also used : HashMap(java.util.HashMap) Config(org.apache.heron.spi.common.Config) Topology(org.apache.heron.api.generated.TopologyAPI.Topology) ExecutorPort(org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort)

Aggregations

ExecutorPort (org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort)5 HashMap (java.util.HashMap)4 LinkedList (java.util.LinkedList)2 Config (org.apache.heron.spi.common.Config)2 ArrayNode (com.fasterxml.jackson.databind.node.ArrayNode)1 ObjectNode (com.fasterxml.jackson.databind.node.ObjectNode)1 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 Quantity (io.kubernetes.client.custom.Quantity)1 V1Patch (io.kubernetes.client.custom.V1Patch)1 ApiClient (io.kubernetes.client.openapi.ApiClient)1 ApiException (io.kubernetes.client.openapi.ApiException)1 Configuration (io.kubernetes.client.openapi.Configuration)1 AppsV1Api (io.kubernetes.client.openapi.apis.AppsV1Api)1 CoreV1Api (io.kubernetes.client.openapi.apis.CoreV1Api)1 V1ConfigMap (io.kubernetes.client.openapi.models.V1ConfigMap)1 V1Container (io.kubernetes.client.openapi.models.V1Container)1 V1ContainerPort (io.kubernetes.client.openapi.models.V1ContainerPort)1 V1EnvVar (io.kubernetes.client.openapi.models.V1EnvVar)1 V1EnvVarSource (io.kubernetes.client.openapi.models.V1EnvVarSource)1 V1LabelSelector (io.kubernetes.client.openapi.models.V1LabelSelector)1