use of io.cdap.cdap.app.runtime.Arguments in project cdap by caskdata.
the class ProgramOptionsCodec method serialize.
@Override
public JsonElement serialize(ProgramOptions src, Type typeOfSrc, JsonSerializationContext context) {
JsonObject json = new JsonObject();
json.add("programId", context.serialize(src.getProgramId(), ProgramId.class));
json.add("arguments", context.serialize(src.getArguments(), Arguments.class));
json.add("userArguments", context.serialize(src.getUserArguments(), Arguments.class));
json.addProperty("debug", src.isDebug());
return json;
}
use of io.cdap.cdap.app.runtime.Arguments in project cdap by caskdata.
the class AbstractProgramTwillRunnable method doInitialize.
/**
* Prepares this instance to execute a program.
*
* @param programOptionFile a json file containing the serialized {@link ProgramOptions}
* @throws Exception if failed to initialize
*/
private void doInitialize(File programOptionFile) throws Exception {
controllerFuture = new CompletableFuture<>();
programCompletion = new CompletableFuture<>();
// Setup process wide settings
Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler());
System.setSecurityManager(new ProgramContainerSecurityManager(System.getSecurityManager()));
SLF4JBridgeHandler.removeHandlersForRootLogger();
SLF4JBridgeHandler.install();
// Create the ProgramOptions
programOptions = createProgramOptions(programOptionFile);
programRunId = programOptions.getProgramId().run(ProgramRunners.getRunId(programOptions));
Arguments systemArgs = programOptions.getArguments();
LoggingContextAccessor.setLoggingContext(LoggingContextHelper.getLoggingContextWithRunId(programRunId, systemArgs.asMap()));
ClusterMode clusterMode = ProgramRunners.getClusterMode(programOptions);
// Loads configurations
Configuration hConf = new Configuration();
if (clusterMode == ClusterMode.ON_PREMISE) {
hConf.clear();
hConf.addResource(new File(systemArgs.getOption(ProgramOptionConstants.HADOOP_CONF_FILE)).toURI().toURL());
}
UserGroupInformation.setConfiguration(hConf);
CConfiguration cConf = CConfiguration.create();
cConf.clear();
cConf.addResource(new File(systemArgs.getOption(ProgramOptionConstants.CDAP_CONF_FILE)).toURI().toURL());
maxStopSeconds = cConf.getLong(io.cdap.cdap.common.conf.Constants.AppFabric.PROGRAM_MAX_STOP_SECONDS);
Injector injector = Guice.createInjector(createModule(cConf, hConf, programOptions, programRunId));
// Initialize log appender
logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
logAppenderInitializer.initialize();
SystemArguments.setLogLevel(programOptions.getUserArguments(), logAppenderInitializer);
// Setup the proxy selector for in active monitoring mode
oldProxySelector = ProxySelector.getDefault();
if (clusterMode == ClusterMode.ISOLATED) {
RuntimeMonitors.setupMonitoring(injector, programOptions);
}
// Create list of core services. They'll will be started in the run method and shutdown when the run
// method completed
coreServices = createCoreServices(injector, programOptions);
// Create the ProgramRunner
programRunner = createProgramRunner(injector);
// Create the Program instance
Location programJarLocation = Locations.toLocation(new File(systemArgs.getOption(ProgramOptionConstants.PROGRAM_JAR)));
ApplicationSpecification appSpec = readJsonFile(new File(systemArgs.getOption(ProgramOptionConstants.APP_SPEC_FILE)), ApplicationSpecification.class);
// Expand the program jar for creating classloader
ClassLoaderFolder classLoaderFolder = BundleJarUtil.prepareClassLoaderFolder(programJarLocation, () -> new File("expanded." + System.currentTimeMillis() + programJarLocation.getName()));
program = Programs.create(cConf, programRunner, new ProgramDescriptor(programOptions.getProgramId(), appSpec), programJarLocation, classLoaderFolder.getDir());
}
use of io.cdap.cdap.app.runtime.Arguments in project cdap by caskdata.
the class DistributedProgramRunner method updateProgramOptions.
/**
* Creates a new instance of {@link ProgramOptions} with artifact localization information and with
* extra system arguments, while maintaining other fields of the given {@link ProgramOptions}.
*
* @param options the original {@link ProgramOptions}.
* @param localizeResources a {@link Map} of {@link LocalizeResource} to be localized to the remote container
* @param tempDir a local temporary directory for creating files for artifact localization.
* @param extraSystemArgs a set of extra system arguments to be added/updated
* @return a new instance of {@link ProgramOptions}
* @throws IOException if failed to create local copy of artifact files
*/
private ProgramOptions updateProgramOptions(ProgramOptions options, Map<String, LocalizeResource> localizeResources, File tempDir, Map<String, String> extraSystemArgs) throws IOException {
Arguments systemArgs = options.getArguments();
Map<String, String> newSystemArgs = new HashMap<>(systemArgs.asMap());
newSystemArgs.putAll(extraSystemArgs);
if (systemArgs.hasOption(ProgramOptionConstants.PLUGIN_ARCHIVE)) {
// If the archive already exists locally, we just need to re-localize it to remote containers
File archiveFile = new File(systemArgs.getOption(ProgramOptionConstants.PLUGIN_ARCHIVE));
// Localize plugins to two files, one expanded into a directory, one not.
localizeResources.put(PLUGIN_DIR, new LocalizeResource(archiveFile, true));
localizeResources.put(PLUGIN_ARCHIVE, new LocalizeResource(archiveFile, false));
} else if (systemArgs.hasOption(ProgramOptionConstants.PLUGIN_DIR)) {
// If there is a plugin directory, then we need to create an archive and localize it to remote containers
File localDir = new File(systemArgs.getOption(ProgramOptionConstants.PLUGIN_DIR));
File archiveFile = new File(tempDir, PLUGIN_DIR + ".jar");
// Store all artifact jars into a new jar file for localization without compression
try (JarOutputStream jarOut = new JarOutputStream(new FileOutputStream(archiveFile))) {
jarOut.setLevel(0);
BundleJarUtil.addToArchive(localDir, jarOut);
}
// Localize plugins to two files, one expanded into a directory, one not.
localizeResources.put(PLUGIN_DIR, new LocalizeResource(archiveFile, true));
localizeResources.put(PLUGIN_ARCHIVE, new LocalizeResource(archiveFile, false));
}
// Add/rename the entries in the system arguments
if (localizeResources.containsKey(PLUGIN_DIR)) {
newSystemArgs.put(ProgramOptionConstants.PLUGIN_DIR, PLUGIN_DIR);
}
if (localizeResources.containsKey(PLUGIN_ARCHIVE)) {
newSystemArgs.put(ProgramOptionConstants.PLUGIN_ARCHIVE, PLUGIN_ARCHIVE);
}
return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(newSystemArgs), options.getUserArguments(), options.isDebug());
}
use of io.cdap.cdap.app.runtime.Arguments in project cdap by caskdata.
the class ProgramRunners method getApplicationPrincipal.
/**
* Returns the application principal if there is one.
*
* @param programOptions the program options to extract information from
* @return the application principal or {@code null} if no application principal is available.
*/
@Nullable
public static KerberosPrincipalId getApplicationPrincipal(ProgramOptions programOptions) {
Arguments systemArgs = programOptions.getArguments();
boolean hasAppPrincipal = Boolean.parseBoolean(systemArgs.getOption(ProgramOptionConstants.APP_PRINCIPAL_EXISTS));
return hasAppPrincipal ? new KerberosPrincipalId(systemArgs.getOption(ProgramOptionConstants.PRINCIPAL)) : null;
}
use of io.cdap.cdap.app.runtime.Arguments in project cdap by caskdata.
the class DistributedProgramContainerModule method getCoreModules.
private List<Module> getCoreModules() {
Arguments systemArgs = programOpts.getArguments();
ClusterMode clusterMode = systemArgs.hasOption(ProgramOptionConstants.CLUSTER_MODE) ? ClusterMode.valueOf(systemArgs.getOption(ProgramOptionConstants.CLUSTER_MODE)) : ClusterMode.ON_PREMISE;
List<Module> modules = new ArrayList<>();
modules.add(new ConfigModule(cConf, hConf));
modules.add(new IOModule());
modules.add(new DFSLocationModule());
modules.add(new MetricsClientRuntimeModule().getDistributedModules());
modules.add(new MessagingClientModule());
modules.add(new AuditModule());
modules.add(new AuthorizationEnforcementModule().getDistributedModules());
modules.add(new SecureStoreClientModule());
modules.add(new MetadataReaderWriterModules().getDistributedModules());
modules.add(new NamespaceQueryAdminModule());
modules.add(new DataSetsModules().getDistributedModules());
modules.add(new AbstractModule() {
@Override
protected void configure() {
bind(ProgramStateWriter.class).to(MessagingProgramStateWriter.class);
bind(WorkflowStateWriter.class).to(MessagingWorkflowStateWriter.class);
// don't need to perform any impersonation from within user programs
bind(UGIProvider.class).to(CurrentUGIProvider.class).in(Scopes.SINGLETON);
// Bind ProgramId to the passed in instance programId so that we can retrieve it back later when needed.
// For example see ProgramDiscoveryExploreClient.
// Also binding to instance is fine here as the programId is guaranteed to not change throughout the
// lifecycle of this program runnable
bind(ProgramId.class).toInstance(programRunId.getParent());
bind(ProgramRunId.class).toInstance(programRunId);
if (serviceAnnouncer != null) {
bind(ServiceAnnouncer.class).toInstance(serviceAnnouncer);
}
bind(PreferencesFetcher.class).to(RemotePreferencesFetcherInternal.class).in(Scopes.SINGLETON);
}
});
addDataFabricModules(modules);
switch(clusterMode) {
case ON_PREMISE:
addOnPremiseModules(modules);
break;
case ISOLATED:
addIsolatedModules(modules);
break;
default:
}
return modules;
}
Aggregations