Search in sources :

Example 6 with PipelineRuntime

use of co.cask.cdap.etl.common.PipelineRuntime in project cdap by caskdata.

the class PipelineAction method run.

@Override
public void run() throws Exception {
    CustomActionContext context = getContext();
    Map<String, String> properties = context.getSpecification().getProperties();
    BatchPhaseSpec phaseSpec = GSON.fromJson(properties.get(Constants.PIPELINEID), BatchPhaseSpec.class);
    PipelinePhase phase = phaseSpec.getPhase();
    StageSpec stageSpec = phase.iterator().next();
    PluginContext pluginContext = new PipelinePluginContext(context, metrics, phaseSpec.isStageLoggingEnabled(), phaseSpec.isProcessTimingEnabled());
    PipelineRuntime pipelineRuntime = new PipelineRuntime(context, metrics);
    Action action = pluginContext.newPluginInstance(stageSpec.getName(), new DefaultMacroEvaluator(pipelineRuntime.getArguments(), context.getLogicalStartTime(), context, context.getNamespace()));
    ActionContext actionContext = new BasicActionContext(context, pipelineRuntime, stageSpec);
    if (!context.getDataTracer(stageSpec.getName()).isEnabled()) {
        action.run(actionContext);
    }
    WorkflowToken token = context.getWorkflowToken();
    if (token == null) {
        throw new IllegalStateException("WorkflowToken cannot be null when action is executed through Workflow.");
    }
    for (Map.Entry<String, String> entry : pipelineRuntime.getArguments().getAddedArguments().entrySet()) {
        token.put(entry.getKey(), entry.getValue());
    }
}
Also used : Action(co.cask.cdap.etl.api.action.Action) CustomAction(co.cask.cdap.api.customaction.CustomAction) AbstractCustomAction(co.cask.cdap.api.customaction.AbstractCustomAction) PipelineRuntime(co.cask.cdap.etl.common.PipelineRuntime) PipelinePluginContext(co.cask.cdap.etl.common.plugin.PipelinePluginContext) PluginContext(co.cask.cdap.api.plugin.PluginContext) WorkflowToken(co.cask.cdap.api.workflow.WorkflowToken) CustomActionContext(co.cask.cdap.api.customaction.CustomActionContext) ActionContext(co.cask.cdap.etl.api.action.ActionContext) PipelinePhase(co.cask.cdap.etl.common.PipelinePhase) StageSpec(co.cask.cdap.etl.spec.StageSpec) DefaultMacroEvaluator(co.cask.cdap.etl.common.DefaultMacroEvaluator) CustomActionContext(co.cask.cdap.api.customaction.CustomActionContext) BatchPhaseSpec(co.cask.cdap.etl.batch.BatchPhaseSpec) HashMap(java.util.HashMap) Map(java.util.Map) PipelinePluginContext(co.cask.cdap.etl.common.plugin.PipelinePluginContext)

Example 7 with PipelineRuntime

use of co.cask.cdap.etl.common.PipelineRuntime in project cdap by caskdata.

the class ETLWorkflow method destroy.

@TransactionPolicy(TransactionControl.EXPLICIT)
@Override
public void destroy() {
    WorkflowContext workflowContext = getContext();
    PipelineRuntime pipelineRuntime = new PipelineRuntime(workflowContext, workflowMetrics);
    if (workflowContext.getDataTracer(PostAction.PLUGIN_TYPE).isEnabled()) {
        return;
    }
    for (Map.Entry<String, PostAction> endingActionEntry : postActions.entrySet()) {
        String name = endingActionEntry.getKey();
        PostAction action = endingActionEntry.getValue();
        StageSpec stageSpec = postActionSpecs.get(name);
        BatchActionContext context = new WorkflowBackedActionContext(workflowContext, pipelineRuntime, stageSpec);
        try {
            action.run(context);
        } catch (Throwable t) {
            LOG.error("Error while running ending action {}.", name, t);
        }
    }
}
Also used : PipelineRuntime(co.cask.cdap.etl.common.PipelineRuntime) BatchActionContext(co.cask.cdap.etl.api.batch.BatchActionContext) WorkflowContext(co.cask.cdap.api.workflow.WorkflowContext) StageSpec(co.cask.cdap.etl.spec.StageSpec) PostAction(co.cask.cdap.etl.api.batch.PostAction) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) Map(java.util.Map) TransactionPolicy(co.cask.cdap.api.annotation.TransactionPolicy)

Example 8 with PipelineRuntime

use of co.cask.cdap.etl.common.PipelineRuntime in project cdap by caskdata.

the class RDDCollection method publishAlerts.

@Override
public void publishAlerts(StageSpec stageSpec, StageStatisticsCollector collector) throws Exception {
    PluginFunctionContext pluginFunctionContext = new PluginFunctionContext(stageSpec, sec, collector);
    AlertPublisher alertPublisher = pluginFunctionContext.createPlugin();
    PipelineRuntime pipelineRuntime = new SparkPipelineRuntime(sec);
    AlertPublisherContext alertPublisherContext = new DefaultAlertPublisherContext(pipelineRuntime, stageSpec, sec.getMessagingContext(), sec.getAdmin());
    alertPublisher.initialize(alertPublisherContext);
    StageMetrics stageMetrics = new DefaultStageMetrics(sec.getMetrics(), stageSpec.getName());
    TrackedIterator<Alert> trackedAlerts = new TrackedIterator<>(((JavaRDD<Alert>) rdd).collect().iterator(), stageMetrics, Constants.Metrics.RECORDS_IN);
    alertPublisher.publish(trackedAlerts);
    alertPublisher.destroy();
}
Also used : PluginFunctionContext(co.cask.cdap.etl.spark.function.PluginFunctionContext) AlertPublisher(co.cask.cdap.etl.api.AlertPublisher) SparkPipelineRuntime(co.cask.cdap.etl.spark.SparkPipelineRuntime) PipelineRuntime(co.cask.cdap.etl.common.PipelineRuntime) SparkPipelineRuntime(co.cask.cdap.etl.spark.SparkPipelineRuntime) TrackedIterator(co.cask.cdap.etl.common.TrackedIterator) Alert(co.cask.cdap.etl.api.Alert) DefaultAlertPublisherContext(co.cask.cdap.etl.common.DefaultAlertPublisherContext) AlertPublisherContext(co.cask.cdap.etl.api.AlertPublisherContext) DefaultAlertPublisherContext(co.cask.cdap.etl.common.DefaultAlertPublisherContext) StageMetrics(co.cask.cdap.etl.api.StageMetrics) DefaultStageMetrics(co.cask.cdap.etl.common.DefaultStageMetrics) DefaultStageMetrics(co.cask.cdap.etl.common.DefaultStageMetrics) JavaRDD(org.apache.spark.api.java.JavaRDD)

Example 9 with PipelineRuntime

use of co.cask.cdap.etl.common.PipelineRuntime in project cdap by caskdata.

the class RDDCollection method store.

@Override
public void store(StageSpec stageSpec, SparkSink<T> sink) throws Exception {
    String stageName = stageSpec.getName();
    PipelineRuntime pipelineRuntime = new SparkPipelineRuntime(sec);
    SparkExecutionPluginContext sparkPluginContext = new BasicSparkExecutionPluginContext(sec, jsc, datasetContext, pipelineRuntime, stageSpec);
    JavaRDD<T> countedRDD = rdd.map(new CountingFunction<T>(stageName, sec.getMetrics(), "records.in", null)).cache();
    sink.run(sparkPluginContext, countedRDD);
}
Also used : SparkExecutionPluginContext(co.cask.cdap.etl.api.batch.SparkExecutionPluginContext) SparkPipelineRuntime(co.cask.cdap.etl.spark.SparkPipelineRuntime) PipelineRuntime(co.cask.cdap.etl.common.PipelineRuntime) SparkPipelineRuntime(co.cask.cdap.etl.spark.SparkPipelineRuntime) CountingFunction(co.cask.cdap.etl.spark.function.CountingFunction)

Example 10 with PipelineRuntime

use of co.cask.cdap.etl.common.PipelineRuntime in project cdap by caskdata.

the class RDDCollection method compute.

@Override
public <U> SparkCollection<U> compute(StageSpec stageSpec, SparkCompute<T, U> compute) throws Exception {
    String stageName = stageSpec.getName();
    PipelineRuntime pipelineRuntime = new SparkPipelineRuntime(sec);
    SparkExecutionPluginContext sparkPluginContext = new BasicSparkExecutionPluginContext(sec, jsc, datasetContext, pipelineRuntime, stageSpec);
    compute.initialize(sparkPluginContext);
    JavaRDD<T> countedInput = rdd.map(new CountingFunction<T>(stageName, sec.getMetrics(), "records.in", null)).cache();
    return wrap(compute.transform(sparkPluginContext, countedInput).map(new CountingFunction<U>(stageName, sec.getMetrics(), "records.out", sec.getDataTracer(stageName))));
}
Also used : SparkExecutionPluginContext(co.cask.cdap.etl.api.batch.SparkExecutionPluginContext) SparkPipelineRuntime(co.cask.cdap.etl.spark.SparkPipelineRuntime) PipelineRuntime(co.cask.cdap.etl.common.PipelineRuntime) SparkPipelineRuntime(co.cask.cdap.etl.spark.SparkPipelineRuntime) CountingFunction(co.cask.cdap.etl.spark.function.CountingFunction)

Aggregations

PipelineRuntime (co.cask.cdap.etl.common.PipelineRuntime)15 DefaultMacroEvaluator (co.cask.cdap.etl.common.DefaultMacroEvaluator)9 MacroEvaluator (co.cask.cdap.api.macro.MacroEvaluator)8 SparkPipelineRuntime (co.cask.cdap.etl.spark.SparkPipelineRuntime)8 StageSpec (co.cask.cdap.etl.spec.StageSpec)8 PluginContext (co.cask.cdap.api.plugin.PluginContext)7 HashMap (java.util.HashMap)6 Map (java.util.Map)6 DatasetContext (co.cask.cdap.api.data.DatasetContext)5 SparkExecutionPluginContext (co.cask.cdap.etl.api.batch.SparkExecutionPluginContext)5 BasicArguments (co.cask.cdap.etl.common.BasicArguments)5 TxRunnable (co.cask.cdap.api.TxRunnable)4 WorkflowToken (co.cask.cdap.api.workflow.WorkflowToken)4 AlertPublisher (co.cask.cdap.etl.api.AlertPublisher)4 SparkPipelinePluginContext (co.cask.cdap.etl.spark.plugin.SparkPipelinePluginContext)4 TransactionPolicy (co.cask.cdap.api.annotation.TransactionPolicy)3 Alert (co.cask.cdap.etl.api.Alert)3 AlertPublisherContext (co.cask.cdap.etl.api.AlertPublisherContext)3 StageMetrics (co.cask.cdap.etl.api.StageMetrics)3 BatchPhaseSpec (co.cask.cdap.etl.batch.BatchPhaseSpec)3