Examples with ExecutorPort org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort used on opensource projects

Example 1 with ExecutorPort

use of org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort in project heron by twitter.

the class MarathonScheduler method getPorts.

protected ArrayNode getPorts(ObjectMapper mapper) {
    ArrayNode ports = mapper.createArrayNode();
    for (Map.Entry<ExecutorPort, String> entry : MarathonConstants.EXECUTOR_PORTS.entrySet()) {
        ObjectNode port = mapper.createObjectNode();
        port.put(MarathonConstants.DOCKER_CONTAINER_PORT, 0);
        port.put(MarathonConstants.PROTOCOL, MarathonConstants.TCP);
        port.put(MarathonConstants.HOST_PORT, 0);
        port.put(MarathonConstants.PORT_NAME, entry.getKey().getName());
        ports.add(port);
    }
    return ports;
}

Example 2 with ExecutorPort

use of org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort in project heron by twitter.

the class V1Controller method getExecutorCommand.

/**
 * Generates the command to start Heron within the <code>container</code>.
 * @param containerId Passed down to <>SchedulerUtils</> to generate executor command.
 * @param numOfInstances Used to configure the debugging ports.
 * @param isExecutor Flag used to generate the correct <code>shard_id</code>.
 * @return The complete command to start Heron in a <code>container</code>.
 */
protected List<String> getExecutorCommand(String containerId, int numOfInstances, boolean isExecutor) {
    final Config configuration = getConfiguration();
    final Config runtimeConfiguration = getRuntimeConfiguration();
    final Map<ExecutorPort, String> ports = KubernetesConstants.EXECUTOR_PORTS.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString()));
    if (TopologyUtils.getTopologyRemoteDebuggingEnabled(Runtime.topology(runtimeConfiguration)) && numOfInstances != 0) {
        List<String> remoteDebuggingPorts = new LinkedList<>();
        IntStream.range(0, numOfInstances).forEach(i -> {
            int port = KubernetesConstants.JVM_REMOTE_DEBUGGER_PORT + i;
            remoteDebuggingPorts.add(String.valueOf(port));
        });
        ports.put(ExecutorPort.JVM_REMOTE_DEBUGGER_PORTS, String.join(",", remoteDebuggingPorts));
    }
    final String[] executorCommand = SchedulerUtils.getExecutorCommand(configuration, runtimeConfiguration, containerId, ports);
    return Arrays.asList("sh", "-c", KubernetesUtils.getConfCommand(configuration) + " && " + KubernetesUtils.getFetchCommand(configuration, runtimeConfiguration) + " && " + setShardIdEnvironmentVariableCommand(isExecutor) + " && " + String.join(" ", executorCommand));
}

Example 3 with ExecutorPort

use of org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort in project heron by twitter.

the class LocalScheduler method getExecutorCommand.

private String[] getExecutorCommand(int container, Set<PackingPlan.InstancePlan> instances) {
    Map<ExecutorPort, String> ports = new HashMap<>();
    for (ExecutorPort executorPort : ExecutorPort.getRequiredPorts()) {
        int port = SysUtils.getFreePort();
        if (port == -1) {
            throw new RuntimeException("Failed to find available ports for executor");
        }
        ports.put(executorPort, String.valueOf(port));
    }
    if (TopologyUtils.getTopologyRemoteDebuggingEnabled(Runtime.topology(runtime)) && instances != null) {
        List<String> remoteDebuggingPorts = new LinkedList<>();
        int portsForRemoteDebugging = instances.size();
        for (int i = 0; i < portsForRemoteDebugging; i++) {
            int port = SysUtils.getFreePort();
            if (port == -1) {
                throw new RuntimeException("Failed to find available ports for executor");
            }
            remoteDebuggingPorts.add(String.valueOf(port));
        }
        ports.put(ExecutorPort.JVM_REMOTE_DEBUGGER_PORTS, String.join(",", remoteDebuggingPorts));
    }
    String[] executorCmd = SchedulerUtils.getExecutorCommand(config, runtime, container, ports);
    LOG.info("Executor command line: " + Arrays.toString(executorCmd));
    return executorCmd;
}

Example 4 with ExecutorPort

use of org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort in project heron by twitter.

the class SlurmScheduler method getExecutorCommand.

protected String[] getExecutorCommand(PackingPlan packing) {
    Map<ExecutorPort, String> ports = new HashMap<>();
    for (ExecutorPort executorPort : ExecutorPort.getRequiredPorts()) {
        int port = SysUtils.getFreePort();
        if (port == -1) {
            throw new RuntimeException("Failed to find available ports for executor");
        }
        ports.put(executorPort, String.valueOf(port));
    }
    String[] executorCmd = SchedulerUtils.executorCommandArgs(this.config, this.runtime, ports, null);
    LOG.log(Level.FINE, "Executor command line: ", Arrays.toString(executorCmd));
    return executorCmd;
}

Example 5 with ExecutorPort

use of org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort in project heron by twitter.

the class HeronExecutorTask method getExecutorCommand.

String[] getExecutorCommand() throws InvalidTopologyException {
    String topologyDefFile = getTopologyDefnFile();
    Topology topology = getTopology(topologyDefFile);
    Config config = SchedulerConfigUtils.loadConfig(cluster, role, env, topologyJar, topologyDefFile, verboseMode, topology);
    Config runtime = Config.newBuilder().put(Key.COMPONENT_RAMMAP, componentRamMap).put(Key.TOPOLOGY_DEFINITION, topology).build();
    Map<ExecutorPort, String> ports = new HashMap<>();
    for (ExecutorPort executorPort : ExecutorPort.getRequiredPorts()) {
        int port = SysUtils.getFreePort();
        if (port == -1) {
            throw new RuntimeException("Failed to find available ports for executor");
        }
        ports.put(executorPort, String.valueOf(port));
    }
    String[] executorCmd = SchedulerUtils.getExecutorCommand(config, runtime, heronExecutorId, ports);
    LOG.info("Executor command line: " + Arrays.toString(executorCmd));
    return executorCmd;
}