use of org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort in project heron by twitter.
the class MarathonScheduler method getPorts.
protected ArrayNode getPorts(ObjectMapper mapper) {
ArrayNode ports = mapper.createArrayNode();
for (Map.Entry<ExecutorPort, String> entry : MarathonConstants.EXECUTOR_PORTS.entrySet()) {
ObjectNode port = mapper.createObjectNode();
port.put(MarathonConstants.DOCKER_CONTAINER_PORT, 0);
port.put(MarathonConstants.PROTOCOL, MarathonConstants.TCP);
port.put(MarathonConstants.HOST_PORT, 0);
port.put(MarathonConstants.PORT_NAME, entry.getKey().getName());
ports.add(port);
}
return ports;
}
use of org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort in project heron by twitter.
the class V1Controller method getExecutorCommand.
/**
* Generates the command to start Heron within the <code>container</code>.
* @param containerId Passed down to <>SchedulerUtils</> to generate executor command.
* @param numOfInstances Used to configure the debugging ports.
* @param isExecutor Flag used to generate the correct <code>shard_id</code>.
* @return The complete command to start Heron in a <code>container</code>.
*/
protected List<String> getExecutorCommand(String containerId, int numOfInstances, boolean isExecutor) {
final Config configuration = getConfiguration();
final Config runtimeConfiguration = getRuntimeConfiguration();
final Map<ExecutorPort, String> ports = KubernetesConstants.EXECUTOR_PORTS.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString()));
if (TopologyUtils.getTopologyRemoteDebuggingEnabled(Runtime.topology(runtimeConfiguration)) && numOfInstances != 0) {
List<String> remoteDebuggingPorts = new LinkedList<>();
IntStream.range(0, numOfInstances).forEach(i -> {
int port = KubernetesConstants.JVM_REMOTE_DEBUGGER_PORT + i;
remoteDebuggingPorts.add(String.valueOf(port));
});
ports.put(ExecutorPort.JVM_REMOTE_DEBUGGER_PORTS, String.join(",", remoteDebuggingPorts));
}
final String[] executorCommand = SchedulerUtils.getExecutorCommand(configuration, runtimeConfiguration, containerId, ports);
return Arrays.asList("sh", "-c", KubernetesUtils.getConfCommand(configuration) + " && " + KubernetesUtils.getFetchCommand(configuration, runtimeConfiguration) + " && " + setShardIdEnvironmentVariableCommand(isExecutor) + " && " + String.join(" ", executorCommand));
}
use of org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort in project heron by twitter.
the class LocalScheduler method getExecutorCommand.
private String[] getExecutorCommand(int container, Set<PackingPlan.InstancePlan> instances) {
Map<ExecutorPort, String> ports = new HashMap<>();
for (ExecutorPort executorPort : ExecutorPort.getRequiredPorts()) {
int port = SysUtils.getFreePort();
if (port == -1) {
throw new RuntimeException("Failed to find available ports for executor");
}
ports.put(executorPort, String.valueOf(port));
}
if (TopologyUtils.getTopologyRemoteDebuggingEnabled(Runtime.topology(runtime)) && instances != null) {
List<String> remoteDebuggingPorts = new LinkedList<>();
int portsForRemoteDebugging = instances.size();
for (int i = 0; i < portsForRemoteDebugging; i++) {
int port = SysUtils.getFreePort();
if (port == -1) {
throw new RuntimeException("Failed to find available ports for executor");
}
remoteDebuggingPorts.add(String.valueOf(port));
}
ports.put(ExecutorPort.JVM_REMOTE_DEBUGGER_PORTS, String.join(",", remoteDebuggingPorts));
}
String[] executorCmd = SchedulerUtils.getExecutorCommand(config, runtime, container, ports);
LOG.info("Executor command line: " + Arrays.toString(executorCmd));
return executorCmd;
}
use of org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort in project heron by twitter.
the class SlurmScheduler method getExecutorCommand.
protected String[] getExecutorCommand(PackingPlan packing) {
Map<ExecutorPort, String> ports = new HashMap<>();
for (ExecutorPort executorPort : ExecutorPort.getRequiredPorts()) {
int port = SysUtils.getFreePort();
if (port == -1) {
throw new RuntimeException("Failed to find available ports for executor");
}
ports.put(executorPort, String.valueOf(port));
}
String[] executorCmd = SchedulerUtils.executorCommandArgs(this.config, this.runtime, ports, null);
LOG.log(Level.FINE, "Executor command line: ", Arrays.toString(executorCmd));
return executorCmd;
}
use of org.apache.heron.scheduler.utils.SchedulerUtils.ExecutorPort in project heron by twitter.
the class HeronExecutorTask method getExecutorCommand.
String[] getExecutorCommand() throws InvalidTopologyException {
String topologyDefFile = getTopologyDefnFile();
Topology topology = getTopology(topologyDefFile);
Config config = SchedulerConfigUtils.loadConfig(cluster, role, env, topologyJar, topologyDefFile, verboseMode, topology);
Config runtime = Config.newBuilder().put(Key.COMPONENT_RAMMAP, componentRamMap).put(Key.TOPOLOGY_DEFINITION, topology).build();
Map<ExecutorPort, String> ports = new HashMap<>();
for (ExecutorPort executorPort : ExecutorPort.getRequiredPorts()) {
int port = SysUtils.getFreePort();
if (port == -1) {
throw new RuntimeException("Failed to find available ports for executor");
}
ports.put(executorPort, String.valueOf(port));
}
String[] executorCmd = SchedulerUtils.getExecutorCommand(config, runtime, heronExecutorId, ports);
LOG.info("Executor command line: " + Arrays.toString(executorCmd));
return executorCmd;
}
Aggregations