Search in sources :

Example 1 with ActionSpec

use of io.cdap.cdap.etl.batch.ActionSpec in project cdap by caskdata.

the class SmartWorkflow method initialize.

@Override
public void initialize(WorkflowContext context) throws Exception {
    super.initialize(context);
    context.enableFieldLineageConsolidation();
    TriggeringScheduleInfo scheduleInfo = context.getTriggeringScheduleInfo();
    if (scheduleInfo != null) {
        String propertiesMappingString = scheduleInfo.getProperties().get(TRIGGERING_PROPERTIES_MAPPING);
        if (propertiesMappingString != null) {
            TriggeringPropertyMapping propertiesMapping = GSON.fromJson(propertiesMappingString, TriggeringPropertyMapping.class);
            updateTokenWithTriggeringProperties(scheduleInfo, propertiesMapping, context.getToken());
        }
    }
    PipelineRuntime pipelineRuntime = new PipelineRuntime(context, workflowMetrics);
    WRAPPERLOGGER.info("Pipeline '{}' is started by user '{}' with arguments {}", context.getApplicationSpecification().getName(), UserGroupInformation.getCurrentUser().getShortUserName(), pipelineRuntime.getArguments().asMap());
    alertPublishers = new HashMap<>();
    postActions = new LinkedHashMap<>();
    spec = GSON.fromJson(context.getWorkflowSpecification().getProperty(Constants.PIPELINE_SPEC_KEY), BatchPipelineSpec.class);
    stageSpecs = new HashMap<>();
    MacroEvaluator macroEvaluator = new DefaultMacroEvaluator(pipelineRuntime.getArguments(), context.getLogicalStartTime(), context, context, context.getNamespace());
    PluginContext pluginContext = new PipelinePluginContext(context, workflowMetrics, spec.isStageLoggingEnabled(), spec.isProcessTimingEnabled());
    for (ActionSpec actionSpec : spec.getEndingActions()) {
        String stageName = actionSpec.getName();
        postActions.put(stageName, pluginContext.newPluginInstance(stageName, macroEvaluator));
        stageSpecs.put(stageName, StageSpec.builder(stageName, actionSpec.getPluginSpec()).setStageLoggingEnabled(spec.isStageLoggingEnabled()).setProcessTimingEnabled(spec.isProcessTimingEnabled()).setMaxPreviewRecords(spec.getNumOfRecordsPreview()).build());
    }
    for (StageSpec stageSpec : spec.getStages()) {
        String stageName = stageSpec.getName();
        stageSpecs.put(stageName, stageSpec);
        if (AlertPublisher.PLUGIN_TYPE.equals(stageSpec.getPluginType())) {
            AlertPublisher alertPublisher = context.newPluginInstance(stageName, macroEvaluator);
            alertPublishers.put(stageName, alertPublisher);
        }
    }
    WRAPPERLOGGER.info("Pipeline '{}' running", context.getApplicationSpecification().getName());
}
Also used : PipelineRuntime(io.cdap.cdap.etl.common.PipelineRuntime) DefaultMacroEvaluator(io.cdap.cdap.etl.common.DefaultMacroEvaluator) MacroEvaluator(io.cdap.cdap.api.macro.MacroEvaluator) ActionSpec(io.cdap.cdap.etl.batch.ActionSpec) AlertPublisher(io.cdap.cdap.etl.api.AlertPublisher) PipelinePluginContext(io.cdap.cdap.etl.common.plugin.PipelinePluginContext) PluginContext(io.cdap.cdap.api.plugin.PluginContext) TriggeringScheduleInfo(io.cdap.cdap.api.schedule.TriggeringScheduleInfo) BatchPipelineSpec(io.cdap.cdap.etl.batch.BatchPipelineSpec) TriggeringPropertyMapping(io.cdap.cdap.etl.proto.v2.TriggeringPropertyMapping) StageSpec(io.cdap.cdap.etl.proto.v2.spec.StageSpec) DefaultMacroEvaluator(io.cdap.cdap.etl.common.DefaultMacroEvaluator) PipelinePluginContext(io.cdap.cdap.etl.common.plugin.PipelinePluginContext)

Aggregations

MacroEvaluator (io.cdap.cdap.api.macro.MacroEvaluator)1 PluginContext (io.cdap.cdap.api.plugin.PluginContext)1 TriggeringScheduleInfo (io.cdap.cdap.api.schedule.TriggeringScheduleInfo)1 AlertPublisher (io.cdap.cdap.etl.api.AlertPublisher)1 ActionSpec (io.cdap.cdap.etl.batch.ActionSpec)1 BatchPipelineSpec (io.cdap.cdap.etl.batch.BatchPipelineSpec)1 DefaultMacroEvaluator (io.cdap.cdap.etl.common.DefaultMacroEvaluator)1 PipelineRuntime (io.cdap.cdap.etl.common.PipelineRuntime)1 PipelinePluginContext (io.cdap.cdap.etl.common.plugin.PipelinePluginContext)1 TriggeringPropertyMapping (io.cdap.cdap.etl.proto.v2.TriggeringPropertyMapping)1 StageSpec (io.cdap.cdap.etl.proto.v2.spec.StageSpec)1