Search in sources :

Example 1 with SparkSubmitter

use of co.cask.cdap.app.runtime.spark.submit.SparkSubmitter in project cdap by caskdata.

the class SparkProgramRunner method run.

@Override
public ProgramController run(Program program, ProgramOptions options) {
    // Get the RunId first. It is used for the creation of the ClassLoader closing thread.
    Arguments arguments = options.getArguments();
    RunId runId = ProgramRunners.getRunId(options);
    Deque<Closeable> closeables = new LinkedList<>();
    try {
        // Extract and verify parameters
        ApplicationSpecification appSpec = program.getApplicationSpecification();
        Preconditions.checkNotNull(appSpec, "Missing application specification.");
        ProgramType processorType = program.getType();
        Preconditions.checkNotNull(processorType, "Missing processor type.");
        Preconditions.checkArgument(processorType == ProgramType.SPARK, "Only Spark process type is supported.");
        SparkSpecification spec = appSpec.getSpark().get(program.getName());
        Preconditions.checkNotNull(spec, "Missing SparkSpecification for %s", program.getName());
        String host = options.getArguments().getOption(ProgramOptionConstants.HOST);
        Preconditions.checkArgument(host != null, "No hostname is provided");
        // Get the WorkflowProgramInfo if it is started by Workflow
        WorkflowProgramInfo workflowInfo = WorkflowProgramInfo.create(arguments);
        DatasetFramework programDatasetFramework = workflowInfo == null ? datasetFramework : NameMappedDatasetFramework.createFromWorkflowProgramInfo(datasetFramework, workflowInfo, appSpec);
        // Setup dataset framework context, if required
        if (programDatasetFramework instanceof ProgramContextAware) {
            ProgramId programId = program.getId();
            ((ProgramContextAware) programDatasetFramework).setContext(new BasicProgramContext(programId.run(runId)));
        }
        PluginInstantiator pluginInstantiator = createPluginInstantiator(options, program.getClassLoader());
        if (pluginInstantiator != null) {
            closeables.addFirst(pluginInstantiator);
        }
        SparkRuntimeContext runtimeContext = new SparkRuntimeContext(new Configuration(hConf), program, options, cConf, host, txClient, programDatasetFramework, discoveryServiceClient, metricsCollectionService, streamAdmin, workflowInfo, pluginInstantiator, secureStore, secureStoreManager, authorizationEnforcer, authenticationContext, messagingService, serviceAnnouncer, pluginFinder, locationFactory);
        closeables.addFirst(runtimeContext);
        Spark spark;
        try {
            spark = new InstantiatorFactory(false).get(TypeToken.of(program.<Spark>getMainClass())).create();
        } catch (Exception e) {
            LOG.error("Failed to instantiate Spark class for {}", spec.getClassName(), e);
            throw Throwables.propagate(e);
        }
        SparkSubmitter submitter = SparkRuntimeContextConfig.isLocal(hConf) ? new LocalSparkSubmitter() : new DistributedSparkSubmitter(hConf, locationFactory, host, runtimeContext, options.getArguments().getOption(Constants.AppFabric.APP_SCHEDULER_QUEUE));
        Service sparkRuntimeService = new SparkRuntimeService(cConf, spark, getPluginArchive(options), runtimeContext, submitter, locationFactory);
        sparkRuntimeService.addListener(createRuntimeServiceListener(closeables), Threads.SAME_THREAD_EXECUTOR);
        ProgramController controller = new SparkProgramController(sparkRuntimeService, runtimeContext);
        LOG.debug("Starting Spark Job. Context: {}", runtimeContext);
        if (SparkRuntimeContextConfig.isLocal(hConf) || UserGroupInformation.isSecurityEnabled()) {
            sparkRuntimeService.start();
        } else {
            ProgramRunners.startAsUser(cConf.get(Constants.CFG_HDFS_USER), sparkRuntimeService);
        }
        return controller;
    } catch (Throwable t) {
        closeAllQuietly(closeables);
        throw Throwables.propagate(t);
    }
}
Also used : ApplicationSpecification(co.cask.cdap.api.app.ApplicationSpecification) SparkSubmitter(co.cask.cdap.app.runtime.spark.submit.SparkSubmitter) DistributedSparkSubmitter(co.cask.cdap.app.runtime.spark.submit.DistributedSparkSubmitter) LocalSparkSubmitter(co.cask.cdap.app.runtime.spark.submit.LocalSparkSubmitter) CConfiguration(co.cask.cdap.common.conf.CConfiguration) Configuration(org.apache.hadoop.conf.Configuration) Closeable(java.io.Closeable) DistributedSparkSubmitter(co.cask.cdap.app.runtime.spark.submit.DistributedSparkSubmitter) NameMappedDatasetFramework(co.cask.cdap.internal.app.runtime.workflow.NameMappedDatasetFramework) DatasetFramework(co.cask.cdap.data2.dataset2.DatasetFramework) InstantiatorFactory(co.cask.cdap.common.lang.InstantiatorFactory) SparkSpecification(co.cask.cdap.api.spark.SparkSpecification) ProgramType(co.cask.cdap.proto.ProgramType) RunId(org.apache.twill.api.RunId) ProgramController(co.cask.cdap.app.runtime.ProgramController) Arguments(co.cask.cdap.app.runtime.Arguments) MessagingService(co.cask.cdap.messaging.MessagingService) MetricsCollectionService(co.cask.cdap.api.metrics.MetricsCollectionService) Service(com.google.common.util.concurrent.Service) ProgramId(co.cask.cdap.proto.id.ProgramId) BasicProgramContext(co.cask.cdap.internal.app.runtime.BasicProgramContext) LinkedList(java.util.LinkedList) IOException(java.io.IOException) WorkflowProgramInfo(co.cask.cdap.internal.app.runtime.workflow.WorkflowProgramInfo) PluginInstantiator(co.cask.cdap.internal.app.runtime.plugin.PluginInstantiator) Spark(co.cask.cdap.api.spark.Spark) LocalSparkSubmitter(co.cask.cdap.app.runtime.spark.submit.LocalSparkSubmitter) ProgramContextAware(co.cask.cdap.data.ProgramContextAware)

Aggregations

ApplicationSpecification (co.cask.cdap.api.app.ApplicationSpecification)1 MetricsCollectionService (co.cask.cdap.api.metrics.MetricsCollectionService)1 Spark (co.cask.cdap.api.spark.Spark)1 SparkSpecification (co.cask.cdap.api.spark.SparkSpecification)1 Arguments (co.cask.cdap.app.runtime.Arguments)1 ProgramController (co.cask.cdap.app.runtime.ProgramController)1 DistributedSparkSubmitter (co.cask.cdap.app.runtime.spark.submit.DistributedSparkSubmitter)1 LocalSparkSubmitter (co.cask.cdap.app.runtime.spark.submit.LocalSparkSubmitter)1 SparkSubmitter (co.cask.cdap.app.runtime.spark.submit.SparkSubmitter)1 CConfiguration (co.cask.cdap.common.conf.CConfiguration)1 InstantiatorFactory (co.cask.cdap.common.lang.InstantiatorFactory)1 ProgramContextAware (co.cask.cdap.data.ProgramContextAware)1 DatasetFramework (co.cask.cdap.data2.dataset2.DatasetFramework)1 BasicProgramContext (co.cask.cdap.internal.app.runtime.BasicProgramContext)1 PluginInstantiator (co.cask.cdap.internal.app.runtime.plugin.PluginInstantiator)1 NameMappedDatasetFramework (co.cask.cdap.internal.app.runtime.workflow.NameMappedDatasetFramework)1 WorkflowProgramInfo (co.cask.cdap.internal.app.runtime.workflow.WorkflowProgramInfo)1 MessagingService (co.cask.cdap.messaging.MessagingService)1 ProgramType (co.cask.cdap.proto.ProgramType)1 ProgramId (co.cask.cdap.proto.id.ProgramId)1