use of io.cdap.cdap.internal.app.runtime.MetricsFieldSetter in project cdap by caskdata.
the class WorkflowDriver method executeCondition.
@SuppressWarnings("unchecked")
private void executeCondition(ApplicationSpecification appSpec, final WorkflowConditionNode node, InstantiatorFactory instantiator, ClassLoader classLoader, WorkflowToken token) throws Exception {
final BasicWorkflowContext context = new BasicWorkflowContext(workflowSpec, token, program, programOptions, cConf, metricsCollectionService, datasetFramework, txClient, discoveryServiceClient, nodeStates, pluginInstantiator, secureStore, secureStoreManager, messagingService, node.getConditionSpecification(), metadataReader, metadataPublisher, namespaceQueryAdmin, fieldLineageWriter, remoteClientFactory);
final Iterator<WorkflowNode> iterator;
Class<?> clz = classLoader.loadClass(node.getPredicateClassName());
Predicate<WorkflowContext> predicate = instantiator.get(TypeToken.of((Class<? extends Predicate<WorkflowContext>>) clz)).create();
if (!(predicate instanceof Condition)) {
iterator = predicate.apply(context) ? node.getIfBranch().iterator() : node.getElseBranch().iterator();
} else {
final Condition workflowCondition = (Condition) predicate;
Reflections.visit(workflowCondition, workflowCondition.getClass(), new PropertyFieldSetter(node.getConditionSpecification().getProperties()), new DataSetFieldSetter(context), new MetricsFieldSetter(context.getMetrics()));
TransactionControl defaultTxControl = workflowContext.getDefaultTxControl();
try {
// AbstractCondition implements final initialize(context) and requires subclass to
// implement initialize(), whereas conditions that directly implement Condition can
// override initialize(context)
TransactionControl txControl = workflowCondition instanceof AbstractCondition ? Transactions.getTransactionControl(defaultTxControl, AbstractCondition.class, workflowCondition, "initialize") : Transactions.getTransactionControl(defaultTxControl, Condition.class, workflowCondition, "initialize", WorkflowContext.class);
context.initializeProgram(workflowCondition, txControl, false);
boolean result = context.execute(() -> workflowCondition.apply(context));
iterator = result ? node.getIfBranch().iterator() : node.getElseBranch().iterator();
} finally {
TransactionControl txControl = Transactions.getTransactionControl(defaultTxControl, Condition.class, workflowCondition, "destroy");
context.destroyProgram(workflowCondition, txControl, false);
}
}
// If a workflow updates its token at a condition node, it will be persisted after the execution of the next node.
// However, the call below ensures that even if the workflow fails/crashes after a condition node, updates from the
// condition node are also persisted.
workflowStateWriter.setWorkflowToken(workflowRunId, token);
executeAll(iterator, appSpec, instantiator, classLoader, token);
}
use of io.cdap.cdap.internal.app.runtime.MetricsFieldSetter in project cdap by caskdata.
the class MapperWrapper method run.
@SuppressWarnings("unchecked")
@Override
public void run(Context context) throws IOException, InterruptedException {
MapReduceClassLoader classLoader = MapReduceClassLoader.getFromConfiguration(context.getConfiguration());
ClassLoader weakReferenceClassLoader = new WeakReferenceDelegatorClassLoader(classLoader);
BasicMapReduceTaskContext basicMapReduceContext = classLoader.getTaskContextProvider().get(context);
String program = basicMapReduceContext.getProgramName();
final MapTaskMetricsWriter mapTaskMetricsWriter = new MapTaskMetricsWriter(basicMapReduceContext.getProgramMetrics(), context);
// this is a hook for periodic flushing of changes buffered by datasets (to avoid OOME)
WrappedMapper.Context flushingContext = createAutoFlushingContext(context, basicMapReduceContext, mapTaskMetricsWriter);
basicMapReduceContext.setHadoopContext(flushingContext);
InputSplit inputSplit = context.getInputSplit();
if (inputSplit instanceof MultiInputTaggedSplit) {
basicMapReduceContext.setInputContext(InputContexts.create((MultiInputTaggedSplit) inputSplit));
}
ClassLoader programClassLoader = classLoader.getProgramClassLoader();
Mapper delegate = createMapperInstance(programClassLoader, getWrappedMapper(context.getConfiguration()), context, program);
// injecting runtime components, like datasets, etc.
try {
Reflections.visit(delegate, delegate.getClass(), new PropertyFieldSetter(basicMapReduceContext.getSpecification().getProperties()), new MetricsFieldSetter(basicMapReduceContext.getMetrics()), new DataSetFieldSetter(basicMapReduceContext));
} catch (Throwable t) {
Throwable rootCause = Throwables.getRootCause(t);
USERLOG.error("Failed to initialize program '{}' with error: {}. Please check the system logs for more details.", program, rootCause.getMessage(), rootCause);
throw new IOException(String.format("Failed to inject fields to %s", delegate.getClass()), t);
}
ClassLoader oldClassLoader;
if (delegate instanceof ProgramLifecycle) {
oldClassLoader = ClassLoaders.setContextClassLoader(weakReferenceClassLoader);
try {
((ProgramLifecycle) delegate).initialize(new MapReduceLifecycleContext(basicMapReduceContext));
} catch (Exception e) {
Throwable rootCause = Throwables.getRootCause(e);
USERLOG.error("Failed to initialize program '{}' with error: {}. Please check the system logs for more " + "details.", program, rootCause.getMessage(), rootCause);
throw new IOException(String.format("Failed to initialize mapper with %s", basicMapReduceContext), e);
} finally {
ClassLoaders.setContextClassLoader(oldClassLoader);
}
}
oldClassLoader = ClassLoaders.setContextClassLoader(weakReferenceClassLoader);
try {
delegate.run(flushingContext);
} finally {
ClassLoaders.setContextClassLoader(oldClassLoader);
}
// memory by tx agent)
try {
basicMapReduceContext.flushOperations();
} catch (Exception e) {
throw new IOException("Failed to flush operations at the end of mapper of " + basicMapReduceContext, e);
}
// Close all writers created by MultipleOutputs
basicMapReduceContext.closeMultiOutputs();
if (delegate instanceof ProgramLifecycle) {
oldClassLoader = ClassLoaders.setContextClassLoader(weakReferenceClassLoader);
try {
((ProgramLifecycle<? extends RuntimeContext>) delegate).destroy();
} catch (Exception e) {
LOG.error("Error during destroy of mapper {}", basicMapReduceContext, e);
// Do nothing, try to finish
} finally {
ClassLoaders.setContextClassLoader(oldClassLoader);
}
}
// Emit metrics one final time
mapTaskMetricsWriter.reportMetrics();
}
use of io.cdap.cdap.internal.app.runtime.MetricsFieldSetter in project cdap by caskdata.
the class CustomActionExecutor method createCustomAction.
@SuppressWarnings("unchecked")
private CustomAction createCustomAction(BasicCustomActionContext context, InstantiatorFactory instantiator, ClassLoader classLoader) throws Exception {
Class<?> clz = Class.forName(context.getSpecification().getClassName(), true, classLoader);
Preconditions.checkArgument(CustomAction.class.isAssignableFrom(clz), "%s is not a CustomAction.", clz);
CustomAction action = instantiator.get(TypeToken.of((Class<? extends CustomAction>) clz)).create();
Reflections.visit(action, action.getClass(), new PropertyFieldSetter(context.getSpecification().getProperties()), new DataSetFieldSetter(context), new MetricsFieldSetter(context.getMetrics()));
return action;
}
use of io.cdap.cdap.internal.app.runtime.MetricsFieldSetter in project cdap by caskdata.
the class WorkflowDriver method initializeWorkflow.
@SuppressWarnings("unchecked")
private Workflow initializeWorkflow() throws Exception {
Class<?> clz = Class.forName(workflowSpec.getClassName(), true, program.getClassLoader());
if (!Workflow.class.isAssignableFrom(clz)) {
throw new IllegalStateException(String.format("%s is not Workflow.", clz));
}
Class<? extends Workflow> workflowClass = (Class<? extends Workflow>) clz;
final Workflow workflow = new InstantiatorFactory(false).get(TypeToken.of(workflowClass)).create();
// set metrics
Reflections.visit(workflow, workflow.getClass(), new MetricsFieldSetter(workflowContext.getMetrics()));
if (!(workflow instanceof ProgramLifecycle)) {
return workflow;
}
final TransactionControl txControl = Transactions.getTransactionControl(workflowContext.getDefaultTxControl(), Workflow.class, workflow, "initialize", WorkflowContext.class);
basicWorkflowToken.setCurrentNode(workflowSpec.getName());
workflowContext.setState(new ProgramState(ProgramStatus.INITIALIZING, null));
workflowContext.initializeProgram((ProgramLifecycle) workflow, txControl, false);
workflowStateWriter.setWorkflowToken(workflowRunId, basicWorkflowToken);
return workflow;
}
use of io.cdap.cdap.internal.app.runtime.MetricsFieldSetter in project cdap by caskdata.
the class WorkerDriver method startUp.
@Override
protected void startUp() throws Exception {
LoggingContextAccessor.setLoggingContext(context.getLoggingContext());
// Instantiate worker instance
Class<?> workerClass = program.getClassLoader().loadClass(spec.getClassName());
@SuppressWarnings("unchecked") TypeToken<Worker> workerType = (TypeToken<Worker>) TypeToken.of(workerClass);
worker = new InstantiatorFactory(false).get(workerType).create();
// Fields injection
Reflections.visit(worker, workerType.getType(), new MetricsFieldSetter(context.getMetrics()), new PropertyFieldSetter(spec.getProperties()));
LOG.debug("Starting Worker Program {}", program.getId());
// Initialize worker
// Worker is always using Explicit transaction
TransactionControl txControl = Transactions.getTransactionControl(TransactionControl.EXPLICIT, Worker.class, worker, "initialize", WorkerContext.class);
context.initializeProgram(worker, txControl, false);
}
Aggregations