use of io.cdap.cdap.app.runtime.ProgramOptions in project cdap by caskdata.
the class AbstractProgramTwillRunnable method createProgramOptions.
/**
* Creates a {@link ProgramOptions} by deserializing the given json file.
*/
private ProgramOptions createProgramOptions(File programOptionsFile) throws IOException {
ProgramOptions original = readJsonFile(programOptionsFile, ProgramOptions.class);
// Overwrite them with environmental information
Map<String, String> arguments = new HashMap<>(original.getArguments().asMap());
arguments.putAll(getExtraSystemArguments());
// Use the name passed in by the constructor as the program name to construct the ProgramId
return new SimpleProgramOptions(original.getProgramId(), new BasicArguments(arguments), original.getUserArguments(), original.isDebug());
}
use of io.cdap.cdap.app.runtime.ProgramOptions in project cdap by caskdata.
the class AbstractInMemoryProgramRunner method startAll.
/**
* Starts all instances of a Program component.
* @param program The program to run
* @param options options for the program
* @param numInstances number of component instances to start
*/
protected final ProgramController startAll(Program program, ProgramOptions options, int numInstances) {
RunId runId = ProgramRunners.getRunId(options);
Table<String, Integer, ProgramController> components = HashBasedTable.create();
try {
for (int instanceId = 0; instanceId < numInstances; instanceId++) {
ProgramOptions componentOptions = createComponentOptions(instanceId, numInstances, runId, options);
ProgramController controller = createProgramRunner().run(program, componentOptions);
components.put(program.getName(), instanceId, controller);
}
return new InMemoryProgramController(components, program, options);
} catch (Throwable t) {
LOG.error("Failed to start all program instances", t);
try {
// Need to stop all started components
Futures.successfulAsList(Iterables.transform(components.values(), new Function<ProgramController, ListenableFuture<?>>() {
@Override
public ListenableFuture<?> apply(ProgramController controller) {
return controller.stop();
}
})).get();
throw Throwables.propagate(t);
} catch (Exception e) {
LOG.error("Failed to stop all program instances upon startup failure.", e);
throw Throwables.propagate(e);
}
}
}
use of io.cdap.cdap.app.runtime.ProgramOptions in project cdap by caskdata.
the class ProvisioningService method provision.
/**
* Record that a cluster will be provisioned for a program run, returning a Runnable that will actually perform
* the cluster provisioning. This method must be run within a transaction.
* The task returned should only be executed after the transaction that ran this method has completed.
* Running the returned Runnable will start the actual task using an executor within this service so that it can be
* tracked and optionally cancelled using {@link #cancelProvisionTask(ProgramRunId)}. The caller does not need to
* submit the runnable using their own executor.
*
* @param provisionRequest the provision request
* @param context context for the transaction
* @return runnable that will actually execute the cluster provisioning
*/
public Runnable provision(ProvisionRequest provisionRequest, StructuredTableContext context) throws IOException {
ProgramRunId programRunId = provisionRequest.getProgramRunId();
ProgramOptions programOptions = provisionRequest.getProgramOptions();
Map<String, String> args = programOptions.getArguments().asMap();
String name = SystemArguments.getProfileProvisioner(args);
Provisioner provisioner = provisionerInfo.get().provisioners.get(name);
// any errors seen here will transition the state straight to deprovisioned since no cluster create was attempted
if (provisioner == null) {
runWithProgramLogging(programRunId, args, () -> LOG.error("Could not provision cluster for the run because provisioner {} does not exist.", name));
programStateWriter.error(programRunId, new IllegalStateException("Provisioner does not exist."));
provisionerNotifier.deprovisioned(programRunId);
return () -> {
};
}
// get plugin requirement information and check for capability to run on the provisioner
Set<PluginRequirement> requirements = GSON.fromJson(args.get(ProgramOptionConstants.PLUGIN_REQUIREMENTS), PLUGIN_REQUIREMENT_SET_TYPE);
if (requirements != null) {
Set<PluginRequirement> unfulfilledRequirements = getUnfulfilledRequirements(provisioner.getCapabilities(), requirements);
if (!unfulfilledRequirements.isEmpty()) {
runWithProgramLogging(programRunId, args, () -> LOG.error(String.format("'%s' cannot be run using profile '%s' because the profile does not met all " + "plugin requirements. Following requirements were not meet by the listed " + "plugins: '%s'", programRunId.getProgram(), name, groupByRequirement(unfulfilledRequirements))));
programStateWriter.error(programRunId, new IllegalArgumentException("Provisioner does not meet all the " + "requirements for the program to run."));
provisionerNotifier.deprovisioned(programRunId);
return () -> {
};
}
}
Map<String, String> properties = SystemArguments.getProfileProperties(args);
ProvisioningOp provisioningOp = new ProvisioningOp(ProvisioningOp.Type.PROVISION, ProvisioningOp.Status.REQUESTING_CREATE);
ProvisioningTaskInfo provisioningTaskInfo = new ProvisioningTaskInfo(programRunId, provisionRequest.getProgramDescriptor(), programOptions, properties, name, provisionRequest.getUser(), provisioningOp, createKeysDirectory(programRunId).toURI(), null);
ProvisionerTable provisionerTable = new ProvisionerTable(context);
provisionerTable.putTaskInfo(provisioningTaskInfo);
return createProvisionTask(provisioningTaskInfo, provisioner);
}
use of io.cdap.cdap.app.runtime.ProgramOptions in project cdap by caskdata.
the class ProvisioningService method createProvisionTask.
private Runnable createProvisionTask(ProvisioningTaskInfo taskInfo, Provisioner provisioner) {
ProgramRunId programRunId = taskInfo.getProgramRunId();
ProgramOptions programOptions = taskInfo.getProgramOptions();
Map<String, String> systemArgs = programOptions.getArguments().asMap();
ProvisionerContext context;
try {
SSHContext sshContext = new DefaultSSHContext(Networks.getAddress(cConf, Constants.NETWORK_PROXY_ADDRESS), locationFactory.create(taskInfo.getSecureKeysDir()), createSSHKeyPair(taskInfo));
context = createContext(cConf, programOptions, programRunId, taskInfo.getUser(), taskInfo.getProvisionerProperties(), sshContext);
} catch (IOException e) {
runWithProgramLogging(taskInfo.getProgramRunId(), systemArgs, () -> LOG.error("Failed to load ssh key. The run will be marked as failed.", e));
programStateWriter.error(programRunId, new IllegalStateException("Failed to load ssh key.", e));
provisionerNotifier.deprovisioning(taskInfo.getProgramRunId());
return () -> {
};
} catch (InvalidMacroException e) {
runWithProgramLogging(taskInfo.getProgramRunId(), systemArgs, () -> LOG.error("Could not evaluate macros while provisoning. " + "The run will be marked as failed.", e));
programStateWriter.error(programRunId, new IllegalStateException("Could not evaluate macros while provisioning", e));
provisionerNotifier.deprovisioning(taskInfo.getProgramRunId());
return () -> {
};
}
// TODO: (CDAP-13246) pick up timeout from profile instead of hardcoding
ProvisioningTask task = new ProvisionTask(taskInfo, transactionRunner, provisioner, context, provisionerNotifier, programStateWriter, 300);
ProvisioningTaskKey taskKey = new ProvisioningTaskKey(programRunId, ProvisioningOp.Type.PROVISION);
return () -> taskExecutor.submit(taskKey, () -> callWithProgramLogging(programRunId, systemArgs, () -> {
try {
return task.executeOnce();
} catch (InterruptedException e) {
LOG.debug("Provision task for program run {} interrupted.", taskInfo.getProgramRunId());
throw e;
} catch (Exception e) {
LOG.info("Provision task for program run {} failed.", taskInfo.getProgramRunId(), e);
throw e;
}
}));
}
use of io.cdap.cdap.app.runtime.ProgramOptions in project cdap by caskdata.
the class ProvisioningServiceTest method createTaskInfo.
private TaskFields createTaskInfo(ProvisionerInfo provisionerInfo) {
ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").workflow("wf").run(RunIds.generate());
Map<String, String> systemArgs = new HashMap<>();
Map<String, String> userArgs = new HashMap<>();
Profile profile = new Profile(ProfileId.NATIVE.getProfile(), "label", "desc", provisionerInfo);
SystemArguments.addProfileArgs(systemArgs, profile);
systemArgs.put(Constants.APP_CDAP_VERSION, APP_CDAP_VERSION);
ProgramOptions programOptions = new SimpleProgramOptions(programRunId.getParent(), new BasicArguments(systemArgs), new BasicArguments(userArgs));
ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
ApplicationSpecification appSpec = new DefaultApplicationSpecification("name", "1.0.0", APP_CDAP_VERSION, "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
ProgramDescriptor programDescriptor = new ProgramDescriptor(programRunId.getParent(), appSpec);
return new TaskFields(programDescriptor, programOptions, programRunId);
}
Aggregations