use of io.cdap.cdap.common.lang.InstantiatorFactory in project cdap by caskdata.
the class ServiceHttpServer method createDelegatorContexts.
@Override
protected List<HandlerDelegatorContext> createDelegatorContexts() throws Exception {
// Constructs all handler delegator. It is for bridging ServiceHttpHandler and HttpHandler (in netty-http).
List<HandlerDelegatorContext> delegatorContexts = new ArrayList<>();
InstantiatorFactory instantiatorFactory = new InstantiatorFactory(false);
for (HttpServiceHandlerSpecification handlerSpec : serviceSpecification.getHandlers().values()) {
Class<?> handlerClass = getProgram().getClassLoader().loadClass(handlerSpec.getClassName());
@SuppressWarnings("unchecked") TypeToken<HttpServiceHandler> type = TypeToken.of((Class<HttpServiceHandler>) handlerClass);
MetricsContext metrics = httpServiceContext.getProgramMetrics().childContext(BasicHttpServiceContext.createMetricsTags(handlerSpec, getInstanceId()));
delegatorContexts.add(new HandlerDelegatorContext(type, instantiatorFactory, handlerSpec, contextFactory, metrics));
}
return delegatorContexts;
}
use of io.cdap.cdap.common.lang.InstantiatorFactory in project cdap by caskdata.
the class WorkflowDriver method run.
@Override
protected void run() throws Exception {
LOG.info("Starting workflow execution for '{}' with Run id '{}'", workflowSpec.getName(), workflowRunId.getRun());
LOG.trace("Workflow specification is {}", workflowSpec);
workflowContext.setState(new ProgramState(ProgramStatus.RUNNING, null));
executeAll(workflowSpec.getNodes().iterator(), program.getApplicationSpecification(), new InstantiatorFactory(false), program.getClassLoader(), basicWorkflowToken);
if (runningThread != null) {
workflowContext.setState(new ProgramState(ProgramStatus.COMPLETED, null));
}
LOG.info("Workflow '{}' with run id '{}' completed", workflowSpec.getName(), workflowRunId.getRun());
}
use of io.cdap.cdap.common.lang.InstantiatorFactory in project cdap by caskdata.
the class SparkProgramRunner method run.
@Override
public ProgramController run(Program program, ProgramOptions options) {
LOG.trace("Starting Spark program {} with SparkProgramRunner of ClassLoader {}", program.getId(), getClass().getClassLoader());
// Get the RunId first. It is used for the creation of the ClassLoader closing thread.
Arguments arguments = options.getArguments();
RunId runId = ProgramRunners.getRunId(options);
Deque<Closeable> closeables = new LinkedList<>();
try {
// Extract and verify parameters
ApplicationSpecification appSpec = program.getApplicationSpecification();
Preconditions.checkNotNull(appSpec, "Missing application specification.");
ProgramType processorType = program.getType();
Preconditions.checkNotNull(processorType, "Missing processor type.");
Preconditions.checkArgument(processorType == ProgramType.SPARK, "Only Spark process type is supported.");
SparkSpecification spec = appSpec.getSpark().get(program.getName());
Preconditions.checkNotNull(spec, "Missing SparkSpecification for %s", program.getName());
String host = options.getArguments().getOption(ProgramOptionConstants.HOST);
Preconditions.checkArgument(host != null, "No hostname is provided");
// Get the WorkflowProgramInfo if it is started by Workflow
WorkflowProgramInfo workflowInfo = WorkflowProgramInfo.create(arguments);
DatasetFramework programDatasetFramework = workflowInfo == null ? datasetFramework : NameMappedDatasetFramework.createFromWorkflowProgramInfo(datasetFramework, workflowInfo, appSpec);
// Setup dataset framework context, if required
if (programDatasetFramework instanceof ProgramContextAware) {
ProgramId programId = program.getId();
((ProgramContextAware) programDatasetFramework).setContext(new BasicProgramContext(programId.run(runId)));
}
PluginInstantiator pluginInstantiator = createPluginInstantiator(options, program.getClassLoader());
if (pluginInstantiator != null) {
closeables.addFirst(pluginInstantiator);
}
SparkRuntimeContext runtimeContext = new SparkRuntimeContext(new Configuration(hConf), program, options, cConf, host, txClient, programDatasetFramework, metricsCollectionService, workflowInfo, pluginInstantiator, secureStore, secureStoreManager, accessEnforcer, authenticationContext, messagingService, serviceAnnouncer, pluginFinder, locationFactory, metadataReader, metadataPublisher, namespaceQueryAdmin, fieldLineageWriter, remoteClientFactory, () -> {
});
closeables.addFirst(runtimeContext);
Spark spark;
try {
spark = new InstantiatorFactory(false).get(TypeToken.of(program.<Spark>getMainClass())).create();
} catch (Exception e) {
LOG.error("Failed to instantiate Spark class for {}", spec.getClassName(), e);
throw Throwables.propagate(e);
}
boolean isLocal = SparkRuntimeContextConfig.isLocal(options);
SparkSubmitter submitter;
// If MasterEnvironment is not available, use non-master env spark submitters
MasterEnvironment masterEnv = MasterEnvironments.getMasterEnvironment();
if (masterEnv != null && cConf.getBoolean(Constants.Environment.PROGRAM_SUBMISSION_MASTER_ENV_ENABLED, true)) {
submitter = new MasterEnvironmentSparkSubmitter(cConf, locationFactory, host, runtimeContext, masterEnv);
} else {
submitter = isLocal ? new LocalSparkSubmitter() : new DistributedSparkSubmitter(hConf, locationFactory, host, runtimeContext, options.getArguments().getOption(Constants.AppFabric.APP_SCHEDULER_QUEUE));
}
Service sparkRuntimeService = new SparkRuntimeService(cConf, spark, getPluginArchive(options), runtimeContext, submitter, locationFactory, isLocal, fieldLineageWriter, masterEnv);
sparkRuntimeService.addListener(createRuntimeServiceListener(closeables), Threads.SAME_THREAD_EXECUTOR);
ProgramController controller = new SparkProgramController(sparkRuntimeService, runtimeContext);
LOG.debug("Starting Spark Job. Context: {}", runtimeContext);
if (isLocal || UserGroupInformation.isSecurityEnabled()) {
sparkRuntimeService.start();
} else {
ProgramRunners.startAsUser(cConf.get(Constants.CFG_HDFS_USER), sparkRuntimeService);
}
return controller;
} catch (Throwable t) {
closeAllQuietly(closeables);
throw Throwables.propagate(t);
}
}
use of io.cdap.cdap.common.lang.InstantiatorFactory in project cdap by caskdata.
the class WorkflowDriver method initializeWorkflow.
@SuppressWarnings("unchecked")
private Workflow initializeWorkflow() throws Exception {
Class<?> clz = Class.forName(workflowSpec.getClassName(), true, program.getClassLoader());
if (!Workflow.class.isAssignableFrom(clz)) {
throw new IllegalStateException(String.format("%s is not Workflow.", clz));
}
Class<? extends Workflow> workflowClass = (Class<? extends Workflow>) clz;
final Workflow workflow = new InstantiatorFactory(false).get(TypeToken.of(workflowClass)).create();
// set metrics
Reflections.visit(workflow, workflow.getClass(), new MetricsFieldSetter(workflowContext.getMetrics()));
if (!(workflow instanceof ProgramLifecycle)) {
return workflow;
}
final TransactionControl txControl = Transactions.getTransactionControl(workflowContext.getDefaultTxControl(), Workflow.class, workflow, "initialize", WorkflowContext.class);
basicWorkflowToken.setCurrentNode(workflowSpec.getName());
workflowContext.setState(new ProgramState(ProgramStatus.INITIALIZING, null));
workflowContext.initializeProgram((ProgramLifecycle) workflow, txControl, false);
workflowStateWriter.setWorkflowToken(workflowRunId, basicWorkflowToken);
return workflow;
}
use of io.cdap.cdap.common.lang.InstantiatorFactory in project cdap by caskdata.
the class WorkerDriver method startUp.
@Override
protected void startUp() throws Exception {
LoggingContextAccessor.setLoggingContext(context.getLoggingContext());
// Instantiate worker instance
Class<?> workerClass = program.getClassLoader().loadClass(spec.getClassName());
@SuppressWarnings("unchecked") TypeToken<Worker> workerType = (TypeToken<Worker>) TypeToken.of(workerClass);
worker = new InstantiatorFactory(false).get(workerType).create();
// Fields injection
Reflections.visit(worker, workerType.getType(), new MetricsFieldSetter(context.getMetrics()), new PropertyFieldSetter(spec.getProperties()));
LOG.debug("Starting Worker Program {}", program.getId());
// Initialize worker
// Worker is always using Explicit transaction
TransactionControl txControl = Transactions.getTransactionControl(TransactionControl.EXPLICIT, Worker.class, worker, "initialize", WorkerContext.class);
context.initializeProgram(worker, txControl, false);
}
Aggregations