use of com.continuuity.weave.internal.ProcessController in project weave by continuuity.
the class ApplicationMasterProcessLauncher method doLaunch.
@Override
@SuppressWarnings("unchecked")
protected <R> ProcessController<R> doLaunch(YarnLaunchContext launchContext) {
final ApplicationId appId = getContainerInfo();
// Set the resource requirement for AM
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(Constants.APP_MASTER_MEMORY_MB);
YarnUtils.setVirtualCores(capability, 1);
// Put in extra environments
Map<String, String> env = ImmutableMap.<String, String>builder().putAll(launchContext.getEnvironment()).put(EnvKeys.YARN_APP_ID, Integer.toString(appId.getId())).put(EnvKeys.YARN_APP_ID_CLUSTER_TIME, Long.toString(appId.getClusterTimestamp())).put(EnvKeys.YARN_APP_ID_STR, appId.toString()).put(EnvKeys.YARN_CONTAINER_MEMORY_MB, Integer.toString(Constants.APP_MASTER_MEMORY_MB)).put(EnvKeys.YARN_CONTAINER_VIRTUAL_CORES, Integer.toString(YarnUtils.getVirtualCores(capability))).build();
launchContext.setEnvironment(env);
return (ProcessController<R>) submitter.submit(launchContext, capability);
}
use of com.continuuity.weave.internal.ProcessController in project weave by continuuity.
the class RunnableProcessLauncher method doLaunch.
@Override
protected <R> ProcessController<R> doLaunch(YarnLaunchContext launchContext) {
Map<String, String> env = Maps.newHashMap(launchContext.getEnvironment());
// Set extra environments
env.put(EnvKeys.YARN_CONTAINER_ID, containerInfo.getId());
env.put(EnvKeys.YARN_CONTAINER_HOST, containerInfo.getHost().getHostName());
env.put(EnvKeys.YARN_CONTAINER_PORT, Integer.toString(containerInfo.getPort()));
env.put(EnvKeys.YARN_CONTAINER_MEMORY_MB, Integer.toString(containerInfo.getMemoryMB()));
env.put(EnvKeys.YARN_CONTAINER_VIRTUAL_CORES, Integer.toString(containerInfo.getVirtualCores()));
launchContext.setEnvironment(env);
LOG.info("Launching in container {}, {}", containerInfo.getId(), launchContext.getCommands());
final Cancellable cancellable = nmClient.start(containerInfo, launchContext);
launched = true;
return new ProcessController<R>() {
@Override
public R getReport() {
// No reporting support for runnable launch yet.
return null;
}
@Override
public void cancel() {
cancellable.cancel();
}
};
}
use of com.continuuity.weave.internal.ProcessController in project weave by continuuity.
the class YarnWeaveRunnerService method prepare.
@Override
public WeavePreparer prepare(WeaveApplication application) {
Preconditions.checkState(isRunning(), "Service not start. Please call start() first.");
final WeaveSpecification weaveSpec = application.configure();
final String appName = weaveSpec.getName();
return new YarnWeavePreparer(yarnConfig, weaveSpec, yarnAppClient, zkClientService, locationFactory, Suppliers.ofInstance(jvmOptions), new YarnWeaveControllerFactory() {
@Override
public YarnWeaveController create(RunId runId, Iterable<LogHandler> logHandlers, Callable<ProcessController<YarnApplicationReport>> startUp) {
ZKClient zkClient = ZKClients.namespace(zkClientService, "/" + appName);
YarnWeaveController controller = listenController(new YarnWeaveController(runId, zkClient, logHandlers, startUp));
synchronized (YarnWeaveRunnerService.this) {
Preconditions.checkArgument(!controllers.contains(appName, runId), "Application %s with runId %s is already running.", appName, runId);
controllers.put(appName, runId, controller);
}
return controller;
}
});
}
use of com.continuuity.weave.internal.ProcessController in project weave by continuuity.
the class YarnWeavePreparer method start.
@Override
public WeaveController start() {
try {
final ProcessLauncher<ApplicationId> launcher = yarnAppClient.createLauncher(user, weaveSpec);
final ApplicationId appId = launcher.getContainerInfo();
Callable<ProcessController<YarnApplicationReport>> submitTask = new Callable<ProcessController<YarnApplicationReport>>() {
@Override
public ProcessController<YarnApplicationReport> call() throws Exception {
String fsUser = locationFactory.getHomeLocation().getName();
// Local files needed by AM
Map<String, LocalFile> localFiles = Maps.newHashMap();
// Local files declared by runnables
Multimap<String, LocalFile> runnableLocalFiles = HashMultimap.create();
String vmOpts = jvmOpts.get();
createAppMasterJar(createBundler(), localFiles);
createContainerJar(createBundler(), localFiles);
populateRunnableLocalFiles(weaveSpec, runnableLocalFiles);
saveWeaveSpec(weaveSpec, runnableLocalFiles, localFiles);
saveLogback(localFiles);
saveLauncher(localFiles);
saveKafka(localFiles);
saveVmOptions(vmOpts, localFiles);
saveArguments(new Arguments(arguments, runnableArgs), localFiles);
saveLocalFiles(localFiles, ImmutableSet.of(Constants.Files.WEAVE_SPEC, Constants.Files.LOGBACK_TEMPLATE, Constants.Files.CONTAINER_JAR, Constants.Files.LAUNCHER_JAR, Constants.Files.ARGUMENTS));
LOG.debug("Submit AM container spec: {}", appId);
// false
return launcher.prepareLaunch(ImmutableMap.<String, String>builder().put(EnvKeys.WEAVE_FS_USER, fsUser).put(EnvKeys.WEAVE_APP_DIR, getAppLocation().toURI().toASCIIString()).put(EnvKeys.WEAVE_ZK_CONNECT, zkClient.getConnectString()).put(EnvKeys.WEAVE_RUN_ID, runId.getId()).put(EnvKeys.WEAVE_RESERVED_MEMORY_MB, Integer.toString(reservedMemory)).put(EnvKeys.WEAVE_APP_NAME, weaveSpec.getName()).build(), localFiles.values(), credentials).noResources().noEnvironment().withCommands().add("java", "-Djava.io.tmpdir=tmp", "-Dyarn.appId=$" + EnvKeys.YARN_APP_ID_STR, "-Dweave.app=$" + EnvKeys.WEAVE_APP_NAME, "-cp", Constants.Files.LAUNCHER_JAR + ":$HADOOP_CONF_DIR", "-Xmx" + (Constants.APP_MASTER_MEMORY_MB - Constants.APP_MASTER_RESERVED_MEMORY_MB) + "m", vmOpts, WeaveLauncher.class.getName(), Constants.Files.APP_MASTER_JAR, ApplicationMasterMain.class.getName(), Boolean.FALSE.toString()).redirectOutput(Constants.STDOUT).redirectError(Constants.STDERR).launch();
}
};
YarnWeaveController controller = controllerFactory.create(runId, logHandlers, submitTask);
controller.start();
return controller;
} catch (Exception e) {
LOG.error("Failed to submit application {}", weaveSpec.getName(), e);
throw Throwables.propagate(e);
}
}
Aggregations