Examples with WorkflowActionNode io.cdap.cdap.api.workflow.WorkflowActionNode used on opensource projects

Example 11 with WorkflowActionNode

use of io.cdap.cdap.api.workflow.WorkflowActionNode in project cdap by cdapio.

the class LineageAdmin method extractAndAddInnerPrograms.

/**
 * Extract inner programs and runs from the workflow run record, the run record's properties have all the
 * inner program run ids. The workflow spec can then be used to determine what the inner programs are and
 * create the program run ids for them
 */
private void extractAndAddInnerPrograms(Set<ProgramId> toVisitPrograms, Map<ProgramRunId, ProgramRunId> programWorkflowMap, Map<ApplicationId, ApplicationSpecification> appSpecs, ProgramRunId programRunId, RunRecordDetail wfRunRecord) {
    ApplicationId appId = programRunId.getParent().getParent();
    WorkflowSpecification workflowSpec = appSpecs.get(appId).getWorkflows().get(programRunId.getProgram());
    Map<String, WorkflowNode> nodeIdMap = workflowSpec.getNodeIdMap();
    wfRunRecord.getProperties().forEach((key, value) -> {
        if (nodeIdMap.containsKey(key)) {
            WorkflowActionNode node = (WorkflowActionNode) nodeIdMap.get(key);
            ProgramType type = ProgramType.valueOf(node.getProgram().getProgramType().name());
            ProgramId program = appId.program(type, key);
            programWorkflowMap.put(program.run(value), programRunId);
            toVisitPrograms.add(program);
        }
    });
}

Example 12 with WorkflowActionNode

use of io.cdap.cdap.api.workflow.WorkflowActionNode in project cdap by cdapio.

the class WorkflowDriver method executeNode.

private void executeNode(ApplicationSpecification appSpec, WorkflowNode node, InstantiatorFactory instantiator, ClassLoader classLoader, WorkflowToken token) throws Exception {
    WorkflowNodeType nodeType = node.getType();
    ((BasicWorkflowToken) token).setCurrentNode(node.getNodeId());
    switch(nodeType) {
        case ACTION:
            WorkflowActionNode actionNode = (WorkflowActionNode) node;
            if (SchedulableProgramType.CUSTOM_ACTION == actionNode.getProgram().getProgramType()) {
                executeCustomAction(actionNode, instantiator, classLoader, token);
            } else {
                executeAction(actionNode, token);
            }
            break;
        case FORK:
            executeFork(appSpec, (WorkflowForkNode) node, instantiator, classLoader, token);
            break;
        case CONDITION:
            executeCondition(appSpec, (WorkflowConditionNode) node, instantiator, classLoader, token);
            break;
        default:
            break;
    }
}

Example 13 with WorkflowActionNode

use of io.cdap.cdap.api.workflow.WorkflowActionNode in project cdap by caskdata.

the class WorkflowDriver method executeNode.

private void executeNode(ApplicationSpecification appSpec, WorkflowNode node, InstantiatorFactory instantiator, ClassLoader classLoader, WorkflowToken token) throws Exception {
    WorkflowNodeType nodeType = node.getType();
    ((BasicWorkflowToken) token).setCurrentNode(node.getNodeId());
    switch(nodeType) {
        case ACTION:
            WorkflowActionNode actionNode = (WorkflowActionNode) node;
            if (SchedulableProgramType.CUSTOM_ACTION == actionNode.getProgram().getProgramType()) {
                executeCustomAction(actionNode, instantiator, classLoader, token);
            } else {
                executeAction(actionNode, token);
            }
            break;
        case FORK:
            executeFork(appSpec, (WorkflowForkNode) node, instantiator, classLoader, token);
            break;
        case CONDITION:
            executeCondition(appSpec, (WorkflowConditionNode) node, instantiator, classLoader, token);
            break;
        default:
            break;
    }
}

Example 14 with WorkflowActionNode

use of io.cdap.cdap.api.workflow.WorkflowActionNode in project cdap by caskdata.

the class ProgramNotificationSubscriberService method processWorkflowOnStop.

/**
 * On workflow program stop, inspects inner program states and adjust them if they are not in end state already.
 *
 * @param appMetadataStore the {@link AppMetadataStore} to write the status to
 * @param programHeartbeatTable the {@link ProgramHeartbeatTable} to write the status to
 * @param programRunId the program run of the completed program
 * @param programRunStatus the status of the completion
 * @param notification the {@link Notification} that carries information about the workflow completion
 * @param sourceId the source message id of the notification
 * @param runnables a {@link List} adding {@link Runnable} to be executed after event handling is completed
 * @throws Exception if failed to update program status
 */
private void processWorkflowOnStop(AppMetadataStore appMetadataStore, ProgramHeartbeatTable programHeartbeatTable, ProgramRunId programRunId, ProgramRunStatus programRunStatus, Notification notification, byte[] sourceId, List<Runnable> runnables) throws Exception {
    ApplicationId appId = programRunId.getParent().getParent();
    WorkflowSpecification workflowSpec = Optional.ofNullable(appMetadataStore.getApplication(appId)).map(appMeta -> appMeta.getSpec().getWorkflows().get(programRunId.getProgram())).orElse(null);
    // If cannot find the workflow spec (e.g. app deleted), then there is nothing we can do.
    if (workflowSpec == null) {
        return;
    }
    // For all MR and Spark nodes, we need to update the inner program run status if they are not in end state yet.
    for (WorkflowNode workflowNode : workflowSpec.getNodeIdMap().values()) {
        if (!(workflowNode instanceof WorkflowActionNode)) {
            continue;
        }
        ScheduleProgramInfo programInfo = ((WorkflowActionNode) workflowNode).getProgram();
        if (!WORKFLOW_INNER_PROGRAM_TYPES.containsKey(programInfo.getProgramType())) {
            continue;
        }
        // Get all active runs of the inner program. If the parent workflow runId is the same as this one,
        // set a terminal state for the inner program run.
        ProgramId innerProgramId = appId.program(WORKFLOW_INNER_PROGRAM_TYPES.get(programInfo.getProgramType()), programInfo.getProgramName());
        Map<ProgramRunId, Notification> innerProgramNotifications = new LinkedHashMap<>();
        appMetadataStore.scanActiveRuns(innerProgramId, runRecord -> {
            Map<String, String> systemArgs = runRecord.getSystemArgs();
            String workflowName = systemArgs.get(ProgramOptionConstants.WORKFLOW_NAME);
            String workflowRun = systemArgs.get(ProgramOptionConstants.WORKFLOW_RUN_ID);
            if (workflowName == null || workflowRun == null) {
                return;
            }
            ProgramRunId workflowRunId = appId.program(ProgramType.WORKFLOW, workflowName).run(workflowRun);
            if (!programRunId.equals(workflowRunId)) {
                return;
            }
            Map<String, String> notificationProps = new HashMap<>(notification.getProperties());
            notificationProps.put(ProgramOptionConstants.PROGRAM_RUN_ID, GSON.toJson(runRecord.getProgramRunId()));
            innerProgramNotifications.put(runRecord.getProgramRunId(), new Notification(Notification.Type.PROGRAM_STATUS, notificationProps));
        });
        for (Map.Entry<ProgramRunId, Notification> entry : innerProgramNotifications.entrySet()) {
            handleProgramEvent(entry.getKey(), programRunStatus, entry.getValue(), sourceId, appMetadataStore, programHeartbeatTable, runnables);
        }
    }
}

Example 15 with WorkflowActionNode

use of io.cdap.cdap.api.workflow.WorkflowActionNode in project cdap by caskdata.

the class DefaultStore method recordCompletedWorkflow.

private void recordCompletedWorkflow(AppMetadataStore metaStore, WorkflowTable workflowTable, WorkflowId workflowId, String runId) throws IOException, TableNotFoundException {
    RunRecordDetail runRecord = metaStore.getRun(workflowId.run(runId));
    if (runRecord == null) {
        return;
    }
    ApplicationId app = workflowId.getParent();
    ApplicationSpecification appSpec = getApplicationSpec(metaStore, app);
    if (appSpec == null || appSpec.getWorkflows() == null || appSpec.getWorkflows().get(workflowId.getProgram()) == null) {
        LOG.warn("Missing ApplicationSpecification for {}, " + "potentially caused by application removal right after stopping workflow {}", app, workflowId);
        return;
    }
    boolean workFlowNodeFailed = false;
    WorkflowSpecification workflowSpec = appSpec.getWorkflows().get(workflowId.getProgram());
    Map<String, WorkflowNode> nodeIdMap = workflowSpec.getNodeIdMap();
    List<WorkflowTable.ProgramRun> programRunsList = new ArrayList<>();
    for (Map.Entry<String, String> entry : runRecord.getProperties().entrySet()) {
        if (!("workflowToken".equals(entry.getKey()) || "runtimeArgs".equals(entry.getKey()) || "workflowNodeState".equals(entry.getKey()))) {
            WorkflowActionNode workflowNode = (WorkflowActionNode) nodeIdMap.get(entry.getKey());
            ProgramType programType = ProgramType.valueOfSchedulableType(workflowNode.getProgram().getProgramType());
            ProgramId innerProgram = app.program(programType, entry.getKey());
            RunRecordDetail innerProgramRun = metaStore.getRun(innerProgram.run(entry.getValue()));
            if (innerProgramRun != null && innerProgramRun.getStatus().equals(ProgramRunStatus.COMPLETED)) {
                Long stopTs = innerProgramRun.getStopTs();
                // since the program is completed, the stop ts cannot be null
                if (stopTs == null) {
                    LOG.warn("Since the program has completed, expected its stop time to not be null. " + "Not writing workflow completed record for Program = {}, Workflow = {}, Run = {}", innerProgram, workflowId, runRecord);
                    workFlowNodeFailed = true;
                    break;
                }
                programRunsList.add(new WorkflowTable.ProgramRun(entry.getKey(), entry.getValue(), programType, stopTs - innerProgramRun.getStartTs()));
            } else {
                workFlowNodeFailed = true;
                break;
            }
        }
    }
    if (workFlowNodeFailed) {
        return;
    }
    workflowTable.write(workflowId, runRecord, programRunsList);
}