Search in sources :

Example 1 with Arguments

use of io.cdap.cdap.app.runtime.Arguments in project cdap by caskdata.

the class ProgramOptionsCodec method serialize.

@Override
public JsonElement serialize(ProgramOptions src, Type typeOfSrc, JsonSerializationContext context) {
    JsonObject json = new JsonObject();
    json.add("programId", context.serialize(src.getProgramId(), ProgramId.class));
    json.add("arguments", context.serialize(src.getArguments(), Arguments.class));
    json.add("userArguments", context.serialize(src.getUserArguments(), Arguments.class));
    json.addProperty("debug", src.isDebug());
    return json;
}
Also used : Arguments(io.cdap.cdap.app.runtime.Arguments) JsonObject(com.google.gson.JsonObject) ProgramId(io.cdap.cdap.proto.id.ProgramId)

Example 2 with Arguments

use of io.cdap.cdap.app.runtime.Arguments in project cdap by caskdata.

the class AbstractProgramTwillRunnable method doInitialize.

/**
 * Prepares this instance to execute a program.
 *
 * @param programOptionFile a json file containing the serialized {@link ProgramOptions}
 * @throws Exception if failed to initialize
 */
private void doInitialize(File programOptionFile) throws Exception {
    controllerFuture = new CompletableFuture<>();
    programCompletion = new CompletableFuture<>();
    // Setup process wide settings
    Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler());
    System.setSecurityManager(new ProgramContainerSecurityManager(System.getSecurityManager()));
    SLF4JBridgeHandler.removeHandlersForRootLogger();
    SLF4JBridgeHandler.install();
    // Create the ProgramOptions
    programOptions = createProgramOptions(programOptionFile);
    programRunId = programOptions.getProgramId().run(ProgramRunners.getRunId(programOptions));
    Arguments systemArgs = programOptions.getArguments();
    LoggingContextAccessor.setLoggingContext(LoggingContextHelper.getLoggingContextWithRunId(programRunId, systemArgs.asMap()));
    ClusterMode clusterMode = ProgramRunners.getClusterMode(programOptions);
    // Loads configurations
    Configuration hConf = new Configuration();
    if (clusterMode == ClusterMode.ON_PREMISE) {
        hConf.clear();
        hConf.addResource(new File(systemArgs.getOption(ProgramOptionConstants.HADOOP_CONF_FILE)).toURI().toURL());
    }
    UserGroupInformation.setConfiguration(hConf);
    CConfiguration cConf = CConfiguration.create();
    cConf.clear();
    cConf.addResource(new File(systemArgs.getOption(ProgramOptionConstants.CDAP_CONF_FILE)).toURI().toURL());
    maxStopSeconds = cConf.getLong(io.cdap.cdap.common.conf.Constants.AppFabric.PROGRAM_MAX_STOP_SECONDS);
    Injector injector = Guice.createInjector(createModule(cConf, hConf, programOptions, programRunId));
    // Initialize log appender
    logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
    logAppenderInitializer.initialize();
    SystemArguments.setLogLevel(programOptions.getUserArguments(), logAppenderInitializer);
    // Setup the proxy selector for in active monitoring mode
    oldProxySelector = ProxySelector.getDefault();
    if (clusterMode == ClusterMode.ISOLATED) {
        RuntimeMonitors.setupMonitoring(injector, programOptions);
    }
    // Create list of core services. They'll will be started in the run method and shutdown when the run
    // method completed
    coreServices = createCoreServices(injector, programOptions);
    // Create the ProgramRunner
    programRunner = createProgramRunner(injector);
    // Create the Program instance
    Location programJarLocation = Locations.toLocation(new File(systemArgs.getOption(ProgramOptionConstants.PROGRAM_JAR)));
    ApplicationSpecification appSpec = readJsonFile(new File(systemArgs.getOption(ProgramOptionConstants.APP_SPEC_FILE)), ApplicationSpecification.class);
    // Expand the program jar for creating classloader
    ClassLoaderFolder classLoaderFolder = BundleJarUtil.prepareClassLoaderFolder(programJarLocation, () -> new File("expanded." + System.currentTimeMillis() + programJarLocation.getName()));
    program = Programs.create(cConf, programRunner, new ProgramDescriptor(programOptions.getProgramId(), appSpec), programJarLocation, classLoaderFolder.getDir());
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) Configuration(org.apache.hadoop.conf.Configuration) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) ClusterMode(io.cdap.cdap.app.guice.ClusterMode) Arguments(io.cdap.cdap.app.runtime.Arguments) SystemArguments(io.cdap.cdap.internal.app.runtime.SystemArguments) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) ClassLoaderFolder(io.cdap.cdap.common.lang.jar.ClassLoaderFolder) LogAppenderInitializer(io.cdap.cdap.logging.appender.LogAppenderInitializer) Injector(com.google.inject.Injector) ProgramDescriptor(io.cdap.cdap.app.program.ProgramDescriptor) UncaughtExceptionHandler(io.cdap.cdap.common.logging.common.UncaughtExceptionHandler) File(java.io.File) Location(org.apache.twill.filesystem.Location)

Example 3 with Arguments

use of io.cdap.cdap.app.runtime.Arguments in project cdap by caskdata.

the class DistributedProgramRunner method updateProgramOptions.

/**
 * Creates a new instance of {@link ProgramOptions} with artifact localization information and with
 * extra system arguments, while maintaining other fields of the given {@link ProgramOptions}.
 *
 * @param options the original {@link ProgramOptions}.
 * @param localizeResources a {@link Map} of {@link LocalizeResource} to be localized to the remote container
 * @param tempDir a local temporary directory for creating files for artifact localization.
 * @param extraSystemArgs a set of extra system arguments to be added/updated
 * @return a new instance of {@link ProgramOptions}
 * @throws IOException if failed to create local copy of artifact files
 */
private ProgramOptions updateProgramOptions(ProgramOptions options, Map<String, LocalizeResource> localizeResources, File tempDir, Map<String, String> extraSystemArgs) throws IOException {
    Arguments systemArgs = options.getArguments();
    Map<String, String> newSystemArgs = new HashMap<>(systemArgs.asMap());
    newSystemArgs.putAll(extraSystemArgs);
    if (systemArgs.hasOption(ProgramOptionConstants.PLUGIN_ARCHIVE)) {
        // If the archive already exists locally, we just need to re-localize it to remote containers
        File archiveFile = new File(systemArgs.getOption(ProgramOptionConstants.PLUGIN_ARCHIVE));
        // Localize plugins to two files, one expanded into a directory, one not.
        localizeResources.put(PLUGIN_DIR, new LocalizeResource(archiveFile, true));
        localizeResources.put(PLUGIN_ARCHIVE, new LocalizeResource(archiveFile, false));
    } else if (systemArgs.hasOption(ProgramOptionConstants.PLUGIN_DIR)) {
        // If there is a plugin directory, then we need to create an archive and localize it to remote containers
        File localDir = new File(systemArgs.getOption(ProgramOptionConstants.PLUGIN_DIR));
        File archiveFile = new File(tempDir, PLUGIN_DIR + ".jar");
        // Store all artifact jars into a new jar file for localization without compression
        try (JarOutputStream jarOut = new JarOutputStream(new FileOutputStream(archiveFile))) {
            jarOut.setLevel(0);
            BundleJarUtil.addToArchive(localDir, jarOut);
        }
        // Localize plugins to two files, one expanded into a directory, one not.
        localizeResources.put(PLUGIN_DIR, new LocalizeResource(archiveFile, true));
        localizeResources.put(PLUGIN_ARCHIVE, new LocalizeResource(archiveFile, false));
    }
    // Add/rename the entries in the system arguments
    if (localizeResources.containsKey(PLUGIN_DIR)) {
        newSystemArgs.put(ProgramOptionConstants.PLUGIN_DIR, PLUGIN_DIR);
    }
    if (localizeResources.containsKey(PLUGIN_ARCHIVE)) {
        newSystemArgs.put(ProgramOptionConstants.PLUGIN_ARCHIVE, PLUGIN_ARCHIVE);
    }
    return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(newSystemArgs), options.getUserArguments(), options.isDebug());
}
Also used : HashMap(java.util.HashMap) FileOutputStream(java.io.FileOutputStream) Arguments(io.cdap.cdap.app.runtime.Arguments) SystemArguments(io.cdap.cdap.internal.app.runtime.SystemArguments) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments) JarOutputStream(java.util.jar.JarOutputStream) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments) File(java.io.File)

Example 4 with Arguments

use of io.cdap.cdap.app.runtime.Arguments in project cdap by caskdata.

the class ProgramRunners method getApplicationPrincipal.

/**
 * Returns the application principal if there is one.
 *
 * @param programOptions the program options to extract information from
 * @return the application principal or {@code null} if no application principal is available.
 */
@Nullable
public static KerberosPrincipalId getApplicationPrincipal(ProgramOptions programOptions) {
    Arguments systemArgs = programOptions.getArguments();
    boolean hasAppPrincipal = Boolean.parseBoolean(systemArgs.getOption(ProgramOptionConstants.APP_PRINCIPAL_EXISTS));
    return hasAppPrincipal ? new KerberosPrincipalId(systemArgs.getOption(ProgramOptionConstants.PRINCIPAL)) : null;
}
Also used : Arguments(io.cdap.cdap.app.runtime.Arguments) KerberosPrincipalId(io.cdap.cdap.proto.id.KerberosPrincipalId) Nullable(javax.annotation.Nullable)

Example 5 with Arguments

use of io.cdap.cdap.app.runtime.Arguments in project cdap by caskdata.

the class DistributedProgramContainerModule method getCoreModules.

private List<Module> getCoreModules() {
    Arguments systemArgs = programOpts.getArguments();
    ClusterMode clusterMode = systemArgs.hasOption(ProgramOptionConstants.CLUSTER_MODE) ? ClusterMode.valueOf(systemArgs.getOption(ProgramOptionConstants.CLUSTER_MODE)) : ClusterMode.ON_PREMISE;
    List<Module> modules = new ArrayList<>();
    modules.add(new ConfigModule(cConf, hConf));
    modules.add(new IOModule());
    modules.add(new DFSLocationModule());
    modules.add(new MetricsClientRuntimeModule().getDistributedModules());
    modules.add(new MessagingClientModule());
    modules.add(new AuditModule());
    modules.add(new AuthorizationEnforcementModule().getDistributedModules());
    modules.add(new SecureStoreClientModule());
    modules.add(new MetadataReaderWriterModules().getDistributedModules());
    modules.add(new NamespaceQueryAdminModule());
    modules.add(new DataSetsModules().getDistributedModules());
    modules.add(new AbstractModule() {

        @Override
        protected void configure() {
            bind(ProgramStateWriter.class).to(MessagingProgramStateWriter.class);
            bind(WorkflowStateWriter.class).to(MessagingWorkflowStateWriter.class);
            // don't need to perform any impersonation from within user programs
            bind(UGIProvider.class).to(CurrentUGIProvider.class).in(Scopes.SINGLETON);
            // Bind ProgramId to the passed in instance programId so that we can retrieve it back later when needed.
            // For example see ProgramDiscoveryExploreClient.
            // Also binding to instance is fine here as the programId is guaranteed to not change throughout the
            // lifecycle of this program runnable
            bind(ProgramId.class).toInstance(programRunId.getParent());
            bind(ProgramRunId.class).toInstance(programRunId);
            if (serviceAnnouncer != null) {
                bind(ServiceAnnouncer.class).toInstance(serviceAnnouncer);
            }
            bind(PreferencesFetcher.class).to(RemotePreferencesFetcherInternal.class).in(Scopes.SINGLETON);
        }
    });
    addDataFabricModules(modules);
    switch(clusterMode) {
        case ON_PREMISE:
            addOnPremiseModules(modules);
            break;
        case ISOLATED:
            addIsolatedModules(modules);
            break;
        default:
    }
    return modules;
}
Also used : IOModule(io.cdap.cdap.common.guice.IOModule) MessagingClientModule(io.cdap.cdap.messaging.guice.MessagingClientModule) NamespaceQueryAdminModule(io.cdap.cdap.common.namespace.guice.NamespaceQueryAdminModule) SecureStoreClientModule(io.cdap.cdap.security.guice.SecureStoreClientModule) ConfigModule(io.cdap.cdap.common.guice.ConfigModule) DataSetsModules(io.cdap.cdap.data.runtime.DataSetsModules) UGIProvider(io.cdap.cdap.security.impersonation.UGIProvider) CurrentUGIProvider(io.cdap.cdap.security.impersonation.CurrentUGIProvider) Arguments(io.cdap.cdap.app.runtime.Arguments) SystemArguments(io.cdap.cdap.internal.app.runtime.SystemArguments) ArrayList(java.util.ArrayList) MetricsClientRuntimeModule(io.cdap.cdap.metrics.guice.MetricsClientRuntimeModule) MessagingProgramStateWriter(io.cdap.cdap.internal.app.program.MessagingProgramStateWriter) AbstractModule(com.google.inject.AbstractModule) MetadataReaderWriterModules(io.cdap.cdap.metadata.MetadataReaderWriterModules) DFSLocationModule(io.cdap.cdap.common.guice.DFSLocationModule) MessagingWorkflowStateWriter(io.cdap.cdap.internal.app.runtime.workflow.MessagingWorkflowStateWriter) PreferencesFetcher(io.cdap.cdap.metadata.PreferencesFetcher) AuditModule(io.cdap.cdap.data2.audit.AuditModule) Module(com.google.inject.Module) SecureStoreClientModule(io.cdap.cdap.security.guice.SecureStoreClientModule) ConfigModule(io.cdap.cdap.common.guice.ConfigModule) NamespaceQueryAdminModule(io.cdap.cdap.common.namespace.guice.NamespaceQueryAdminModule) SystemDatasetRuntimeModule(io.cdap.cdap.data.runtime.SystemDatasetRuntimeModule) CoreSecurityModule(io.cdap.cdap.security.guice.CoreSecurityModule) DFSLocationModule(io.cdap.cdap.common.guice.DFSLocationModule) KafkaLogAppenderModule(io.cdap.cdap.logging.guice.KafkaLogAppenderModule) MessagingClientModule(io.cdap.cdap.messaging.guice.MessagingClientModule) RemoteLogAppenderModule(io.cdap.cdap.logging.guice.RemoteLogAppenderModule) CoreSecurityRuntimeModule(io.cdap.cdap.security.guice.CoreSecurityRuntimeModule) AuthorizationEnforcementModule(io.cdap.cdap.security.authorization.AuthorizationEnforcementModule) KafkaClientModule(io.cdap.cdap.common.guice.KafkaClientModule) ZKClientModule(io.cdap.cdap.common.guice.ZKClientModule) TMSLogAppenderModule(io.cdap.cdap.logging.guice.TMSLogAppenderModule) AuditModule(io.cdap.cdap.data2.audit.AuditModule) IOModule(io.cdap.cdap.common.guice.IOModule) ZKDiscoveryModule(io.cdap.cdap.common.guice.ZKDiscoveryModule) MetricsClientRuntimeModule(io.cdap.cdap.metrics.guice.MetricsClientRuntimeModule) AbstractModule(com.google.inject.AbstractModule) AuthorizationEnforcementModule(io.cdap.cdap.security.authorization.AuthorizationEnforcementModule)

Aggregations

Arguments (io.cdap.cdap.app.runtime.Arguments)13 ProgramId (io.cdap.cdap.proto.id.ProgramId)7 SystemArguments (io.cdap.cdap.internal.app.runtime.SystemArguments)6 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)5 BasicArguments (io.cdap.cdap.internal.app.runtime.BasicArguments)5 ApplicationSpecification (io.cdap.cdap.api.app.ApplicationSpecification)4 SimpleProgramOptions (io.cdap.cdap.internal.app.runtime.SimpleProgramOptions)4 Service (com.google.common.util.concurrent.Service)3 Injector (com.google.inject.Injector)3 MetricsCollectionService (io.cdap.cdap.api.metrics.MetricsCollectionService)3 ProgramController (io.cdap.cdap.app.runtime.ProgramController)3 HashMap (java.util.HashMap)3 JsonObject (com.google.gson.JsonObject)2 ClusterMode (io.cdap.cdap.app.guice.ClusterMode)2 ProgramDescriptor (io.cdap.cdap.app.program.ProgramDescriptor)2 ProgramOptions (io.cdap.cdap.app.runtime.ProgramOptions)2 InstantiatorFactory (io.cdap.cdap.common.lang.InstantiatorFactory)2 UncaughtExceptionHandler (io.cdap.cdap.common.logging.common.UncaughtExceptionHandler)2 ProgramContextAware (io.cdap.cdap.data.ProgramContextAware)2 DatasetFramework (io.cdap.cdap.data2.dataset2.DatasetFramework)2