Search in sources :

Example 6 with InstantiatorFactory

use of io.cdap.cdap.common.lang.InstantiatorFactory in project cdap by cdapio.

the class SparkProgramRunner method run.

@Override
public ProgramController run(Program program, ProgramOptions options) {
    LOG.trace("Starting Spark program {} with SparkProgramRunner of ClassLoader {}", program.getId(), getClass().getClassLoader());
    // Get the RunId first. It is used for the creation of the ClassLoader closing thread.
    Arguments arguments = options.getArguments();
    RunId runId = ProgramRunners.getRunId(options);
    Deque<Closeable> closeables = new LinkedList<>();
    try {
        // Extract and verify parameters
        ApplicationSpecification appSpec = program.getApplicationSpecification();
        Preconditions.checkNotNull(appSpec, "Missing application specification.");
        ProgramType processorType = program.getType();
        Preconditions.checkNotNull(processorType, "Missing processor type.");
        Preconditions.checkArgument(processorType == ProgramType.SPARK, "Only Spark process type is supported.");
        SparkSpecification spec = appSpec.getSpark().get(program.getName());
        Preconditions.checkNotNull(spec, "Missing SparkSpecification for %s", program.getName());
        String host = options.getArguments().getOption(ProgramOptionConstants.HOST);
        Preconditions.checkArgument(host != null, "No hostname is provided");
        // Get the WorkflowProgramInfo if it is started by Workflow
        WorkflowProgramInfo workflowInfo = WorkflowProgramInfo.create(arguments);
        DatasetFramework programDatasetFramework = workflowInfo == null ? datasetFramework : NameMappedDatasetFramework.createFromWorkflowProgramInfo(datasetFramework, workflowInfo, appSpec);
        // Setup dataset framework context, if required
        if (programDatasetFramework instanceof ProgramContextAware) {
            ProgramId programId = program.getId();
            ((ProgramContextAware) programDatasetFramework).setContext(new BasicProgramContext(programId.run(runId)));
        }
        PluginInstantiator pluginInstantiator = createPluginInstantiator(options, program.getClassLoader());
        if (pluginInstantiator != null) {
            closeables.addFirst(pluginInstantiator);
        }
        SparkRuntimeContext runtimeContext = new SparkRuntimeContext(new Configuration(hConf), program, options, cConf, host, txClient, programDatasetFramework, metricsCollectionService, workflowInfo, pluginInstantiator, secureStore, secureStoreManager, accessEnforcer, authenticationContext, messagingService, serviceAnnouncer, pluginFinder, locationFactory, metadataReader, metadataPublisher, namespaceQueryAdmin, fieldLineageWriter, remoteClientFactory, () -> {
        });
        closeables.addFirst(runtimeContext);
        Spark spark;
        try {
            spark = new InstantiatorFactory(false).get(TypeToken.of(program.<Spark>getMainClass())).create();
        } catch (Exception e) {
            LOG.error("Failed to instantiate Spark class for {}", spec.getClassName(), e);
            throw Throwables.propagate(e);
        }
        boolean isLocal = SparkRuntimeContextConfig.isLocal(options);
        SparkSubmitter submitter;
        // If MasterEnvironment is not available, use non-master env spark submitters
        MasterEnvironment masterEnv = MasterEnvironments.getMasterEnvironment();
        if (masterEnv != null && cConf.getBoolean(Constants.Environment.PROGRAM_SUBMISSION_MASTER_ENV_ENABLED, true)) {
            submitter = new MasterEnvironmentSparkSubmitter(cConf, locationFactory, host, runtimeContext, masterEnv);
        } else {
            submitter = isLocal ? new LocalSparkSubmitter() : new DistributedSparkSubmitter(hConf, locationFactory, host, runtimeContext, options.getArguments().getOption(Constants.AppFabric.APP_SCHEDULER_QUEUE));
        }
        Service sparkRuntimeService = new SparkRuntimeService(cConf, spark, getPluginArchive(options), runtimeContext, submitter, locationFactory, isLocal, fieldLineageWriter, masterEnv);
        sparkRuntimeService.addListener(createRuntimeServiceListener(closeables), Threads.SAME_THREAD_EXECUTOR);
        ProgramController controller = new SparkProgramController(sparkRuntimeService, runtimeContext);
        LOG.debug("Starting Spark Job. Context: {}", runtimeContext);
        if (isLocal || UserGroupInformation.isSecurityEnabled()) {
            sparkRuntimeService.start();
        } else {
            ProgramRunners.startAsUser(cConf.get(Constants.CFG_HDFS_USER), sparkRuntimeService);
        }
        return controller;
    } catch (Throwable t) {
        closeAllQuietly(closeables);
        throw Throwables.propagate(t);
    }
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) MasterEnvironmentSparkSubmitter(io.cdap.cdap.app.runtime.spark.submit.MasterEnvironmentSparkSubmitter) MasterEnvironmentSparkSubmitter(io.cdap.cdap.app.runtime.spark.submit.MasterEnvironmentSparkSubmitter) LocalSparkSubmitter(io.cdap.cdap.app.runtime.spark.submit.LocalSparkSubmitter) SparkSubmitter(io.cdap.cdap.app.runtime.spark.submit.SparkSubmitter) DistributedSparkSubmitter(io.cdap.cdap.app.runtime.spark.submit.DistributedSparkSubmitter) Configuration(org.apache.hadoop.conf.Configuration) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) Closeable(java.io.Closeable) DistributedSparkSubmitter(io.cdap.cdap.app.runtime.spark.submit.DistributedSparkSubmitter) DatasetFramework(io.cdap.cdap.data2.dataset2.DatasetFramework) NameMappedDatasetFramework(io.cdap.cdap.internal.app.runtime.workflow.NameMappedDatasetFramework) InstantiatorFactory(io.cdap.cdap.common.lang.InstantiatorFactory) SparkSpecification(io.cdap.cdap.api.spark.SparkSpecification) ProgramType(io.cdap.cdap.proto.ProgramType) RunId(org.apache.twill.api.RunId) ProgramController(io.cdap.cdap.app.runtime.ProgramController) Arguments(io.cdap.cdap.app.runtime.Arguments) MessagingService(io.cdap.cdap.messaging.MessagingService) Service(com.google.common.util.concurrent.Service) MetricsCollectionService(io.cdap.cdap.api.metrics.MetricsCollectionService) ProgramId(io.cdap.cdap.proto.id.ProgramId) BasicProgramContext(io.cdap.cdap.internal.app.runtime.BasicProgramContext) LinkedList(java.util.LinkedList) IOException(java.io.IOException) WorkflowProgramInfo(io.cdap.cdap.internal.app.runtime.workflow.WorkflowProgramInfo) MasterEnvironment(io.cdap.cdap.master.spi.environment.MasterEnvironment) PluginInstantiator(io.cdap.cdap.internal.app.runtime.plugin.PluginInstantiator) Spark(io.cdap.cdap.api.spark.Spark) LocalSparkSubmitter(io.cdap.cdap.app.runtime.spark.submit.LocalSparkSubmitter) ProgramContextAware(io.cdap.cdap.data.ProgramContextAware)

Example 7 with InstantiatorFactory

use of io.cdap.cdap.common.lang.InstantiatorFactory in project cdap by cdapio.

the class ServiceHttpServer method initializeService.

@Override
protected void initializeService() throws Exception {
    super.initializeService();
    // Instantiate service instance
    Class<?> serviceClass = program.getClassLoader().loadClass(serviceSpecification.getClassName());
    @SuppressWarnings("unchecked") TypeToken<Service> serviceType = (TypeToken<Service>) TypeToken.of(serviceClass);
    service = new InstantiatorFactory(false).get(serviceType).create();
    // Initialize service
    // Service is always using Explicit transaction
    TransactionControl txControl = Transactions.getTransactionControl(TransactionControl.EXPLICIT, Service.class, service, "initialize", ServiceContext.class);
    serviceContext.initializeProgram(service, txControl, false);
}
Also used : InstantiatorFactory(io.cdap.cdap.common.lang.InstantiatorFactory) TypeToken(com.google.common.reflect.TypeToken) TransactionControl(io.cdap.cdap.api.annotation.TransactionControl) AbstractSystemService(io.cdap.cdap.api.service.AbstractSystemService) NettyHttpService(io.cdap.http.NettyHttpService) Service(io.cdap.cdap.api.service.Service) MessagingService(io.cdap.cdap.messaging.MessagingService) MetricsCollectionService(io.cdap.cdap.api.metrics.MetricsCollectionService)

Example 8 with InstantiatorFactory

use of io.cdap.cdap.common.lang.InstantiatorFactory in project cdap by cdapio.

the class WorkflowDriver method initializeWorkflow.

@SuppressWarnings("unchecked")
private Workflow initializeWorkflow() throws Exception {
    Class<?> clz = Class.forName(workflowSpec.getClassName(), true, program.getClassLoader());
    if (!Workflow.class.isAssignableFrom(clz)) {
        throw new IllegalStateException(String.format("%s is not Workflow.", clz));
    }
    Class<? extends Workflow> workflowClass = (Class<? extends Workflow>) clz;
    final Workflow workflow = new InstantiatorFactory(false).get(TypeToken.of(workflowClass)).create();
    // set metrics
    Reflections.visit(workflow, workflow.getClass(), new MetricsFieldSetter(workflowContext.getMetrics()));
    if (!(workflow instanceof ProgramLifecycle)) {
        return workflow;
    }
    final TransactionControl txControl = Transactions.getTransactionControl(workflowContext.getDefaultTxControl(), Workflow.class, workflow, "initialize", WorkflowContext.class);
    basicWorkflowToken.setCurrentNode(workflowSpec.getName());
    workflowContext.setState(new ProgramState(ProgramStatus.INITIALIZING, null));
    workflowContext.initializeProgram((ProgramLifecycle) workflow, txControl, false);
    workflowStateWriter.setWorkflowToken(workflowRunId, basicWorkflowToken);
    return workflow;
}
Also used : InstantiatorFactory(io.cdap.cdap.common.lang.InstantiatorFactory) MetricsFieldSetter(io.cdap.cdap.internal.app.runtime.MetricsFieldSetter) ProgramLifecycle(io.cdap.cdap.api.ProgramLifecycle) TransactionControl(io.cdap.cdap.api.annotation.TransactionControl) Workflow(io.cdap.cdap.api.workflow.Workflow) ProgramState(io.cdap.cdap.api.ProgramState)

Example 9 with InstantiatorFactory

use of io.cdap.cdap.common.lang.InstantiatorFactory in project cdap by cdapio.

the class WorkerDriver method startUp.

@Override
protected void startUp() throws Exception {
    LoggingContextAccessor.setLoggingContext(context.getLoggingContext());
    // Instantiate worker instance
    Class<?> workerClass = program.getClassLoader().loadClass(spec.getClassName());
    @SuppressWarnings("unchecked") TypeToken<Worker> workerType = (TypeToken<Worker>) TypeToken.of(workerClass);
    worker = new InstantiatorFactory(false).get(workerType).create();
    // Fields injection
    Reflections.visit(worker, workerType.getType(), new MetricsFieldSetter(context.getMetrics()), new PropertyFieldSetter(spec.getProperties()));
    LOG.debug("Starting Worker Program {}", program.getId());
    // Initialize worker
    // Worker is always using Explicit transaction
    TransactionControl txControl = Transactions.getTransactionControl(TransactionControl.EXPLICIT, Worker.class, worker, "initialize", WorkerContext.class);
    context.initializeProgram(worker, txControl, false);
}
Also used : InstantiatorFactory(io.cdap.cdap.common.lang.InstantiatorFactory) PropertyFieldSetter(io.cdap.cdap.common.lang.PropertyFieldSetter) MetricsFieldSetter(io.cdap.cdap.internal.app.runtime.MetricsFieldSetter) TypeToken(com.google.common.reflect.TypeToken) TransactionControl(io.cdap.cdap.api.annotation.TransactionControl) Worker(io.cdap.cdap.api.worker.Worker)

Example 10 with InstantiatorFactory

use of io.cdap.cdap.common.lang.InstantiatorFactory in project cdap by caskdata.

the class WorkflowDriver method initializeWorkflow.

@SuppressWarnings("unchecked")
private Workflow initializeWorkflow() throws Exception {
    Class<?> clz = Class.forName(workflowSpec.getClassName(), true, program.getClassLoader());
    if (!Workflow.class.isAssignableFrom(clz)) {
        throw new IllegalStateException(String.format("%s is not Workflow.", clz));
    }
    Class<? extends Workflow> workflowClass = (Class<? extends Workflow>) clz;
    final Workflow workflow = new InstantiatorFactory(false).get(TypeToken.of(workflowClass)).create();
    // set metrics
    Reflections.visit(workflow, workflow.getClass(), new MetricsFieldSetter(workflowContext.getMetrics()));
    if (!(workflow instanceof ProgramLifecycle)) {
        return workflow;
    }
    final TransactionControl txControl = Transactions.getTransactionControl(workflowContext.getDefaultTxControl(), Workflow.class, workflow, "initialize", WorkflowContext.class);
    basicWorkflowToken.setCurrentNode(workflowSpec.getName());
    workflowContext.setState(new ProgramState(ProgramStatus.INITIALIZING, null));
    workflowContext.initializeProgram((ProgramLifecycle) workflow, txControl, false);
    workflowStateWriter.setWorkflowToken(workflowRunId, basicWorkflowToken);
    return workflow;
}
Also used : InstantiatorFactory(io.cdap.cdap.common.lang.InstantiatorFactory) MetricsFieldSetter(io.cdap.cdap.internal.app.runtime.MetricsFieldSetter) ProgramLifecycle(io.cdap.cdap.api.ProgramLifecycle) TransactionControl(io.cdap.cdap.api.annotation.TransactionControl) Workflow(io.cdap.cdap.api.workflow.Workflow) ProgramState(io.cdap.cdap.api.ProgramState)

Aggregations

InstantiatorFactory (io.cdap.cdap.common.lang.InstantiatorFactory)16 TransactionControl (io.cdap.cdap.api.annotation.TransactionControl)6 MetricsCollectionService (io.cdap.cdap.api.metrics.MetricsCollectionService)6 MetricsFieldSetter (io.cdap.cdap.internal.app.runtime.MetricsFieldSetter)6 MessagingService (io.cdap.cdap.messaging.MessagingService)6 ArrayList (java.util.ArrayList)6 TypeToken (com.google.common.reflect.TypeToken)4 Service (com.google.common.util.concurrent.Service)4 ProgramState (io.cdap.cdap.api.ProgramState)4 ApplicationSpecification (io.cdap.cdap.api.app.ApplicationSpecification)4 MetricsContext (io.cdap.cdap.api.metrics.MetricsContext)4 Arguments (io.cdap.cdap.app.runtime.Arguments)4 ProgramController (io.cdap.cdap.app.runtime.ProgramController)4 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)4 PropertyFieldSetter (io.cdap.cdap.common.lang.PropertyFieldSetter)4 ProgramContextAware (io.cdap.cdap.data.ProgramContextAware)4 DatasetFramework (io.cdap.cdap.data2.dataset2.DatasetFramework)4 BasicProgramContext (io.cdap.cdap.internal.app.runtime.BasicProgramContext)4 PluginInstantiator (io.cdap.cdap.internal.app.runtime.plugin.PluginInstantiator)4 NameMappedDatasetFramework (io.cdap.cdap.internal.app.runtime.workflow.NameMappedDatasetFramework)4