Examples with NodeValue io.cdap.cdap.api.workflow.NodeValue used on opensource projects

Example 1 with NodeValue

use of io.cdap.cdap.api.workflow.NodeValue in project cdap by caskdata.

the class SmartWorkflow method destroy.

@Override
public void destroy() {
    WorkflowContext workflowContext = getContext();
    PipelineRuntime pipelineRuntime = new PipelineRuntime(workflowContext, workflowMetrics);
    // Execute the post actions only if pipeline is not running in preview mode.
    if (!workflowContext.getDataTracer(PostAction.PLUGIN_TYPE).isEnabled()) {
        for (Map.Entry<String, PostAction> endingActionEntry : postActions.entrySet()) {
            String name = endingActionEntry.getKey();
            PostAction action = endingActionEntry.getValue();
            StageSpec stageSpec = stageSpecs.get(name);
            BatchActionContext context = new WorkflowBackedActionContext(workflowContext, pipelineRuntime, stageSpec);
            try {
                action.run(context);
            } catch (Throwable t) {
                LOG.error("Error while running post action {}.", name, t);
            }
        }
    }
    Map<String, String> connectorDatasets = GSON.fromJson(workflowContext.getWorkflowSpecification().getProperty(Constants.CONNECTOR_DATASETS), STAGE_DATASET_MAP);
    // publish all alerts
    for (Map.Entry<String, AlertPublisher> alertPublisherEntry : alertPublishers.entrySet()) {
        String stageName = alertPublisherEntry.getKey();
        AlertPublisher alertPublisher = alertPublisherEntry.getValue();
        FileSet alertConnector = workflowContext.getDataset(connectorDatasets.get(stageName));
        try (CloseableIterator<Alert> alerts = new AlertReader(alertConnector)) {
            if (!alerts.hasNext()) {
                continue;
            }
            StageMetrics stageMetrics = new DefaultStageMetrics(workflowMetrics, stageName);
            StageSpec stageSpec = stageSpecs.get(stageName);
            AlertPublisherContext alertContext = new DefaultAlertPublisherContext(pipelineRuntime, stageSpec, workflowContext, workflowContext.getAdmin());
            alertPublisher.initialize(alertContext);
            TrackedIterator<Alert> trackedIterator = new TrackedIterator<>(alerts, stageMetrics, Constants.Metrics.RECORDS_IN);
            alertPublisher.publish(trackedIterator);
        } catch (Exception e) {
            LOG.warn("Stage {} had errors publishing alerts. Alerts may not have been published.", stageName, e);
        } finally {
            try {
                alertPublisher.destroy();
            } catch (Exception e) {
                LOG.warn("Error destroying alert publisher for stage {}", stageName, e);
            }
        }
    }
    ProgramStatus status = getContext().getState().getStatus();
    if (status == ProgramStatus.FAILED) {
        WRAPPERLOGGER.error("Pipeline '{}' failed.", getContext().getApplicationSpecification().getName());
    } else {
        WRAPPERLOGGER.info("Pipeline '{}' {}.", getContext().getApplicationSpecification().getName(), status == ProgramStatus.COMPLETED ? "succeeded" : status.name().toLowerCase());
    }
    MacroEvaluator macroEvaluator = new DefaultMacroEvaluator(pipelineRuntime.getArguments(), workflowContext.getLogicalStartTime(), workflowContext, workflowContext, workflowContext.getNamespace());
    // Get resolved plugin properties
    Map<String, Map<String, String>> resolvedProperties = new HashMap<>();
    for (StageSpec spec : stageSpecs.values()) {
        String stageName = spec.getName();
        resolvedProperties.put(stageName, workflowContext.getPluginProperties(stageName, macroEvaluator).getProperties());
    }
    // Add resolved plugin properties to workflow token as a JSON String
    workflowContext.getToken().put(RESOLVED_PLUGIN_PROPERTIES_MAP, GSON.toJson(resolvedProperties));
    // record only if the Workflow is successful
    if (status != ProgramStatus.COMPLETED) {
        return;
    }
    // Collect field operations from each phase
    WorkflowToken token = workflowContext.getToken();
    List<NodeValue> allNodeValues = token.getAll(Constants.FIELD_OPERATION_KEY_IN_WORKFLOW_TOKEN);
    if (allNodeValues.isEmpty()) {
        // no field lineage recorded by any stage
        return;
    }
    Map<String, List<FieldOperation>> allStageOperations = new HashMap<>();
    for (StageSpec stageSpec : stageSpecs.values()) {
        allStageOperations.put(stageSpec.getName(), new ArrayList<>());
    }
    for (NodeValue nodeValue : allNodeValues) {
        Map<String, List<FieldOperation>> stageOperations = GSON.fromJson(nodeValue.getValue().toString(), STAGE_OPERATIONS_MAP);
        for (Map.Entry<String, List<FieldOperation>> entry : stageOperations.entrySet()) {
            // ignore them
            if (allStageOperations.containsKey(entry.getKey())) {
                allStageOperations.get(entry.getKey()).addAll(entry.getValue());
            }
        }
    }
    FieldLineageProcessor processor = new FieldLineageProcessor(spec);
    Set<Operation> processedOperations = processor.validateAndConvert(allStageOperations);
    if (!processedOperations.isEmpty()) {
        workflowContext.record(processedOperations);
    }
}

Example 2 with NodeValue

use of io.cdap.cdap.api.workflow.NodeValue in project cdap by caskdata.

the class WorkflowTokenDetail method of.

/**
 * Constructs a {@link WorkflowTokenDetail} by flattening the {@link Value} from the output of
 * {@link WorkflowToken#getAll(WorkflowToken.Scope)}.
 *
 * @param data the workflow token data to flatten
 * @return {@link WorkflowTokenDetail} from the specified workflow token data
 */
public static WorkflowTokenDetail of(Map<String, List<NodeValue>> data) {
    Map<String, List<NodeValueDetail>> converted = new HashMap<>();
    for (Map.Entry<String, List<NodeValue>> entry : data.entrySet()) {
        List<NodeValueDetail> convertedList = new ArrayList<>();
        for (NodeValue nodeValue : entry.getValue()) {
            convertedList.add(new NodeValueDetail(nodeValue.getNodeName(), nodeValue.getValue().toString()));
        }
        converted.put(entry.getKey(), convertedList);
    }
    return new WorkflowTokenDetail(converted);
}

Example 3 with NodeValue

use of io.cdap.cdap.api.workflow.NodeValue in project cdap by caskdata.

the class BasicWorkflowToken method put.

private void put(String key, Value value, Scope scope) {
    if (!putAllowed) {
        String msg = String.format("Failed to put key '%s' from node '%s' in the WorkflowToken. Put operation is not " + "allowed from the Mapper and Reducer classes and from Spark executor.", key, nodeName);
        throw new UnsupportedOperationException(msg);
    }
    Preconditions.checkNotNull(key, "Null key cannot be added in the WorkflowToken.");
    Preconditions.checkNotNull(value, String.format("Null value provided for the key '%s'.", key));
    Preconditions.checkNotNull(value.toString(), String.format("Null value provided for the key '%s'.", key));
    Preconditions.checkState(nodeName != null, "nodeName cannot be null.");
    List<NodeValue> nodeValueList = tokenValueMap.get(scope).get(key);
    if (nodeValueList == null) {
        nodeValueList = Lists.newArrayList();
        tokenValueMap.get(scope).put(key, nodeValueList);
    }
    NodeValue nodeValueToAddUpdate = new NodeValue(nodeName, value);
    // In that case replace that entry with the new one
    for (int i = 0; i < nodeValueList.size(); i++) {
        NodeValue existingNodeValue = nodeValueList.get(i);
        if (existingNodeValue.getNodeName().equals(nodeName)) {
            addOrUpdate(key, nodeValueToAddUpdate, nodeValueList, i);
            return;
        }
    }
    addOrUpdate(key, nodeValueToAddUpdate, nodeValueList, -1);
}