Search in sources :

Example 1 with ProgramStatus

use of io.cdap.cdap.api.ProgramStatus in project cdap by caskdata.

the class SmartWorkflow method destroy.

@Override
public void destroy() {
    WorkflowContext workflowContext = getContext();
    PipelineRuntime pipelineRuntime = new PipelineRuntime(workflowContext, workflowMetrics);
    // Execute the post actions only if pipeline is not running in preview mode.
    if (!workflowContext.getDataTracer(PostAction.PLUGIN_TYPE).isEnabled()) {
        for (Map.Entry<String, PostAction> endingActionEntry : postActions.entrySet()) {
            String name = endingActionEntry.getKey();
            PostAction action = endingActionEntry.getValue();
            StageSpec stageSpec = stageSpecs.get(name);
            BatchActionContext context = new WorkflowBackedActionContext(workflowContext, pipelineRuntime, stageSpec);
            try {
                action.run(context);
            } catch (Throwable t) {
                LOG.error("Error while running post action {}.", name, t);
            }
        }
    }
    Map<String, String> connectorDatasets = GSON.fromJson(workflowContext.getWorkflowSpecification().getProperty(Constants.CONNECTOR_DATASETS), STAGE_DATASET_MAP);
    // publish all alerts
    for (Map.Entry<String, AlertPublisher> alertPublisherEntry : alertPublishers.entrySet()) {
        String stageName = alertPublisherEntry.getKey();
        AlertPublisher alertPublisher = alertPublisherEntry.getValue();
        FileSet alertConnector = workflowContext.getDataset(connectorDatasets.get(stageName));
        try (CloseableIterator<Alert> alerts = new AlertReader(alertConnector)) {
            if (!alerts.hasNext()) {
                continue;
            }
            StageMetrics stageMetrics = new DefaultStageMetrics(workflowMetrics, stageName);
            StageSpec stageSpec = stageSpecs.get(stageName);
            AlertPublisherContext alertContext = new DefaultAlertPublisherContext(pipelineRuntime, stageSpec, workflowContext, workflowContext.getAdmin());
            alertPublisher.initialize(alertContext);
            TrackedIterator<Alert> trackedIterator = new TrackedIterator<>(alerts, stageMetrics, Constants.Metrics.RECORDS_IN);
            alertPublisher.publish(trackedIterator);
        } catch (Exception e) {
            LOG.warn("Stage {} had errors publishing alerts. Alerts may not have been published.", stageName, e);
        } finally {
            try {
                alertPublisher.destroy();
            } catch (Exception e) {
                LOG.warn("Error destroying alert publisher for stage {}", stageName, e);
            }
        }
    }
    ProgramStatus status = getContext().getState().getStatus();
    if (status == ProgramStatus.FAILED) {
        WRAPPERLOGGER.error("Pipeline '{}' failed.", getContext().getApplicationSpecification().getName());
    } else {
        WRAPPERLOGGER.info("Pipeline '{}' {}.", getContext().getApplicationSpecification().getName(), status == ProgramStatus.COMPLETED ? "succeeded" : status.name().toLowerCase());
    }
    MacroEvaluator macroEvaluator = new DefaultMacroEvaluator(pipelineRuntime.getArguments(), workflowContext.getLogicalStartTime(), workflowContext, workflowContext, workflowContext.getNamespace());
    // Get resolved plugin properties
    Map<String, Map<String, String>> resolvedProperties = new HashMap<>();
    for (StageSpec spec : stageSpecs.values()) {
        String stageName = spec.getName();
        resolvedProperties.put(stageName, workflowContext.getPluginProperties(stageName, macroEvaluator).getProperties());
    }
    // Add resolved plugin properties to workflow token as a JSON String
    workflowContext.getToken().put(RESOLVED_PLUGIN_PROPERTIES_MAP, GSON.toJson(resolvedProperties));
    // record only if the Workflow is successful
    if (status != ProgramStatus.COMPLETED) {
        return;
    }
    // Collect field operations from each phase
    WorkflowToken token = workflowContext.getToken();
    List<NodeValue> allNodeValues = token.getAll(Constants.FIELD_OPERATION_KEY_IN_WORKFLOW_TOKEN);
    if (allNodeValues.isEmpty()) {
        // no field lineage recorded by any stage
        return;
    }
    Map<String, List<FieldOperation>> allStageOperations = new HashMap<>();
    for (StageSpec stageSpec : stageSpecs.values()) {
        allStageOperations.put(stageSpec.getName(), new ArrayList<>());
    }
    for (NodeValue nodeValue : allNodeValues) {
        Map<String, List<FieldOperation>> stageOperations = GSON.fromJson(nodeValue.getValue().toString(), STAGE_OPERATIONS_MAP);
        for (Map.Entry<String, List<FieldOperation>> entry : stageOperations.entrySet()) {
            // ignore them
            if (allStageOperations.containsKey(entry.getKey())) {
                allStageOperations.get(entry.getKey()).addAll(entry.getValue());
            }
        }
    }
    FieldLineageProcessor processor = new FieldLineageProcessor(spec);
    Set<Operation> processedOperations = processor.validateAndConvert(allStageOperations);
    if (!processedOperations.isEmpty()) {
        workflowContext.record(processedOperations);
    }
}
Also used : NodeValue(io.cdap.cdap.api.workflow.NodeValue) PipelineRuntime(io.cdap.cdap.etl.common.PipelineRuntime) DefaultMacroEvaluator(io.cdap.cdap.etl.common.DefaultMacroEvaluator) MacroEvaluator(io.cdap.cdap.api.macro.MacroEvaluator) BatchActionContext(io.cdap.cdap.etl.api.batch.BatchActionContext) WorkflowBackedActionContext(io.cdap.cdap.etl.batch.WorkflowBackedActionContext) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) WorkflowToken(io.cdap.cdap.api.workflow.WorkflowToken) FieldOperation(io.cdap.cdap.etl.api.lineage.field.FieldOperation) Operation(io.cdap.cdap.api.lineage.field.Operation) AlertReader(io.cdap.cdap.etl.batch.connector.AlertReader) StageSpec(io.cdap.cdap.etl.proto.v2.spec.StageSpec) DefaultMacroEvaluator(io.cdap.cdap.etl.common.DefaultMacroEvaluator) List(java.util.List) ArrayList(java.util.ArrayList) LinkedList(java.util.LinkedList) StageMetrics(io.cdap.cdap.etl.api.StageMetrics) DefaultStageMetrics(io.cdap.cdap.etl.common.DefaultStageMetrics) AlertPublisherContext(io.cdap.cdap.etl.api.AlertPublisherContext) DefaultAlertPublisherContext(io.cdap.cdap.etl.common.DefaultAlertPublisherContext) FieldLineageProcessor(io.cdap.cdap.etl.lineage.FieldLineageProcessor) AlertPublisher(io.cdap.cdap.etl.api.AlertPublisher) FileSet(io.cdap.cdap.api.dataset.lib.FileSet) TrackedIterator(io.cdap.cdap.etl.common.TrackedIterator) WorkflowContext(io.cdap.cdap.api.workflow.WorkflowContext) DisjointConnectionsException(io.cdap.cdap.etl.planner.DisjointConnectionsException) ValidationException(io.cdap.cdap.etl.api.validation.ValidationException) Alert(io.cdap.cdap.etl.api.Alert) PostAction(io.cdap.cdap.etl.api.batch.PostAction) DefaultAlertPublisherContext(io.cdap.cdap.etl.common.DefaultAlertPublisherContext) Map(java.util.Map) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) DefaultStageMetrics(io.cdap.cdap.etl.common.DefaultStageMetrics) ProgramStatus(io.cdap.cdap.api.ProgramStatus)

Example 2 with ProgramStatus

use of io.cdap.cdap.api.ProgramStatus in project cdap by caskdata.

the class MapReduceRuntimeService method emitFieldLineage.

private boolean emitFieldLineage() {
    if (context.getFieldLineageOperations().isEmpty()) {
        // no operations to emit
        return false;
    }
    ProgramStatus status = context.getState().getStatus();
    if (ProgramStatus.COMPLETED != status) {
        // MapReduce program failed, so field operations wont be emitted
        return false;
    }
    WorkflowProgramInfo workflowInfo = context.getWorkflowInfo();
    return workflowInfo == null || !workflowInfo.fieldLineageConsolidationEnabled();
}
Also used : WorkflowProgramInfo(io.cdap.cdap.internal.app.runtime.workflow.WorkflowProgramInfo) ProgramStatus(io.cdap.cdap.api.ProgramStatus)

Example 3 with ProgramStatus

use of io.cdap.cdap.api.ProgramStatus in project cdap by caskdata.

the class ProgramStatusTrigger method getTriggerSatisfiedResult.

/**
 * Helper method to return a result from the given supplier if the trigger is satisfied with the given notifications,
 * or return the default result if the trigger is not satisfied.
 *
 * @param notifications notifications used to determine whether the trigger is satisfied
 * @param defaultResult the default result to return if the trigger is not satisfied
 * @param function the function to get result from if the trigger is satisfied
 * @param <T> type of the result to be returned
 * @return a result of type T
 */
private <T> T getTriggerSatisfiedResult(List<Notification> notifications, T defaultResult, Function<ProgramRunInfo, T> function) {
    for (Notification notification : notifications) {
        if (!Notification.Type.PROGRAM_STATUS.equals(notification.getNotificationType())) {
            continue;
        }
        String programRunIdString = notification.getProperties().get(ProgramOptionConstants.PROGRAM_RUN_ID);
        String programRunStatusString = notification.getProperties().get(ProgramOptionConstants.PROGRAM_STATUS);
        // Ignore notifications which specify an invalid programRunId or programStatus
        if (programRunIdString == null || programRunStatusString == null) {
            continue;
        }
        ProgramStatus programStatus;
        try {
            programStatus = ProgramRunStatus.toProgramStatus(ProgramRunStatus.valueOf(programRunStatusString));
        } catch (IllegalArgumentException e) {
            // Return silently, this happens for statuses that are not meant to be scheduled
            continue;
        }
        ProgramRunId programRunId = GSON.fromJson(programRunIdString, ProgramRunId.class);
        ProgramId triggeringProgramId = programRunId.getParent();
        if (this.programId.equals(triggeringProgramId) && programStatuses.contains(programStatus)) {
            return function.apply(new ProgramRunInfo(programStatus, programRunId));
        }
    }
    return defaultResult;
}
Also used : ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) ProgramId(io.cdap.cdap.proto.id.ProgramId) Notification(io.cdap.cdap.proto.Notification) ProgramStatus(io.cdap.cdap.api.ProgramStatus)

Example 4 with ProgramStatus

use of io.cdap.cdap.api.ProgramStatus in project cdap by caskdata.

the class ProgramScheduleStoreDataset method modifySchedulesTriggeredByDeletedProgram.

/**
 * Update all schedules that can be triggered by the given deleted program. A schedule will be removed if
 * the only {@link ProgramStatusTrigger} in it is triggered by the deleted program. Schedules with composite triggers
 * will be updated if the composite trigger can still be satisfied after the program is deleted, otherwise the
 * schedules will be deleted.
 *
 * @param programId the program id for which to delete the schedules
 * @return the IDs of the schedules that were deleted
 */
public List<ProgramSchedule> modifySchedulesTriggeredByDeletedProgram(ProgramId programId) throws IOException {
    long deleteTime = System.currentTimeMillis();
    List<ProgramSchedule> deleted = new ArrayList<>();
    Set<ProgramScheduleRecord> scheduleRecords = new HashSet<>();
    for (ProgramStatus status : ProgramStatus.values()) {
        scheduleRecords.addAll(findSchedules(Schedulers.triggerKeyForProgramStatus(programId, status)));
    }
    for (ProgramScheduleRecord scheduleRecord : scheduleRecords) {
        ProgramSchedule schedule = scheduleRecord.getSchedule();
        markScheduleAsDeleted(schedule.getScheduleId(), deleteTime);
        triggerStore.deleteAll(Range.singleton(getScheduleKeys(schedule.getScheduleId())));
        if (schedule.getTrigger() instanceof AbstractSatisfiableCompositeTrigger) {
            // get the updated composite trigger by removing the program status trigger of the given program
            Trigger updatedTrigger = ((AbstractSatisfiableCompositeTrigger) schedule.getTrigger()).getTriggerWithDeletedProgram(programId);
            if (updatedTrigger == null) {
                deleted.add(schedule);
                continue;
            }
            // if the updated composite trigger is not null, add the schedule back with updated composite trigger
            try {
                addScheduleWithStatus(new ProgramSchedule(schedule.getName(), schedule.getDescription(), schedule.getProgramId(), schedule.getProperties(), updatedTrigger, schedule.getConstraints(), schedule.getTimeoutMillis()), scheduleRecord.getMeta().getStatus(), System.currentTimeMillis());
            } catch (AlreadyExistsException e) {
                // this should never happen
                LOG.warn("Failed to add the schedule '{}' triggered by '{}' with updated trigger '{}', " + "skip adding this schedule.", schedule.getScheduleId(), programId, updatedTrigger, e);
            }
        } else {
            deleted.add(schedule);
        }
    }
    return deleted;
}
Also used : ProgramStatusTrigger(io.cdap.cdap.internal.app.runtime.schedule.trigger.ProgramStatusTrigger) SatisfiableTrigger(io.cdap.cdap.internal.app.runtime.schedule.trigger.SatisfiableTrigger) Trigger(io.cdap.cdap.api.schedule.Trigger) AbstractSatisfiableCompositeTrigger(io.cdap.cdap.internal.app.runtime.schedule.trigger.AbstractSatisfiableCompositeTrigger) AlreadyExistsException(io.cdap.cdap.common.AlreadyExistsException) ProgramSchedule(io.cdap.cdap.internal.app.runtime.schedule.ProgramSchedule) ArrayList(java.util.ArrayList) AbstractSatisfiableCompositeTrigger(io.cdap.cdap.internal.app.runtime.schedule.trigger.AbstractSatisfiableCompositeTrigger) ProgramScheduleRecord(io.cdap.cdap.internal.app.runtime.schedule.ProgramScheduleRecord) HashSet(java.util.HashSet) ProgramStatus(io.cdap.cdap.api.ProgramStatus)

Example 5 with ProgramStatus

use of io.cdap.cdap.api.ProgramStatus in project cdap by caskdata.

the class DataStreamsSparkLauncher method destroy.

@TransactionPolicy(TransactionControl.EXPLICIT)
@Override
public void destroy() {
    super.destroy();
    ProgramStatus status = getContext().getState().getStatus();
    WRAPPERLOGGER.info("Pipeline '{}' {}", getContext().getApplicationSpecification().getName(), status == ProgramStatus.COMPLETED ? "succeeded" : status.name().toLowerCase());
}
Also used : ProgramStatus(io.cdap.cdap.api.ProgramStatus) TransactionPolicy(io.cdap.cdap.api.annotation.TransactionPolicy)

Aggregations

ProgramStatus (io.cdap.cdap.api.ProgramStatus)6 WorkflowProgramInfo (io.cdap.cdap.internal.app.runtime.workflow.WorkflowProgramInfo)2 ArrayList (java.util.ArrayList)2 TransactionPolicy (io.cdap.cdap.api.annotation.TransactionPolicy)1 FileSet (io.cdap.cdap.api.dataset.lib.FileSet)1 Operation (io.cdap.cdap.api.lineage.field.Operation)1 MacroEvaluator (io.cdap.cdap.api.macro.MacroEvaluator)1 Trigger (io.cdap.cdap.api.schedule.Trigger)1 NodeValue (io.cdap.cdap.api.workflow.NodeValue)1 WorkflowContext (io.cdap.cdap.api.workflow.WorkflowContext)1 WorkflowToken (io.cdap.cdap.api.workflow.WorkflowToken)1 AlreadyExistsException (io.cdap.cdap.common.AlreadyExistsException)1 Alert (io.cdap.cdap.etl.api.Alert)1 AlertPublisher (io.cdap.cdap.etl.api.AlertPublisher)1 AlertPublisherContext (io.cdap.cdap.etl.api.AlertPublisherContext)1 StageMetrics (io.cdap.cdap.etl.api.StageMetrics)1 BatchActionContext (io.cdap.cdap.etl.api.batch.BatchActionContext)1 PostAction (io.cdap.cdap.etl.api.batch.PostAction)1 FieldOperation (io.cdap.cdap.etl.api.lineage.field.FieldOperation)1 ValidationException (io.cdap.cdap.etl.api.validation.ValidationException)1