Examples with WorkflowSpecification io.cdap.cdap.api.workflow.WorkflowSpecification used on opensource projects

Example 16 with WorkflowSpecification

use of io.cdap.cdap.api.workflow.WorkflowSpecification in project cdap by cdapio.

the class SystemMetadataWriterStageTest method testCapabilityTags.

@Test
public void testCapabilityTags() throws Exception {
    String appName = CapabilityAppWithWorkflow.class.getSimpleName();
    ApplicationId appId = NamespaceId.DEFAULT.app(appName);
    String[] capabilityTestNames = { "cdc", "healthcare" };
    Requirements requirements = new Requirements(Collections.emptySet(), Stream.of(capabilityTestNames).collect(Collectors.toSet()));
    ApplicationClass applicationClass = new ApplicationClass(CapabilityAppWithWorkflow.class.getName(), appName, null, requirements);
    String workflowName = CapabilityAppWithWorkflow.SampleWorkflow.class.getSimpleName();
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact(appId.getApplication(), "1.0");
    ApplicationWithPrograms appWithPrograms = createAppWithWorkflow(artifactId, appId, workflowName, new CapabilityAppWithWorkflow(), applicationClass);
    WorkflowSpecification workflowSpec = appWithPrograms.getSpecification().getWorkflows().get(workflowName);
    MetadataWriterStage systemMetadataWriterStage = new MetadataWriterStage(metadataServiceClient);
    StageContext stageContext = new StageContext(Object.class);
    systemMetadataWriterStage.process(stageContext);
    systemMetadataWriterStage.process(appWithPrograms);
    Assert.assertEquals(false, metadataStorage.read(new Read(appId.toMetadataEntity(), MetadataScope.SYSTEM, MetadataKind.PROPERTY)).isEmpty());
    // Test that all test capabilities are present in the metadata
    Map<String, String> metadataProperties = metadataStorage.read(new Read(appId.toMetadataEntity())).getProperties(MetadataScope.SYSTEM);
    Set<String> capabilityNames = Arrays.stream(metadataProperties.get(AppSystemMetadataWriter.CAPABILITY_TAG).split(AppSystemMetadataWriter.CAPABILITY_DELIMITER)).collect(Collectors.toSet());
    Assert.assertEquals(Arrays.stream(capabilityTestNames).collect(Collectors.toSet()), capabilityNames);
}

Example 17 with WorkflowSpecification

use of io.cdap.cdap.api.workflow.WorkflowSpecification in project cdap by cdapio.

the class SystemMetadataWriterStageTest method testWorkflowTags.

@Test
public void testWorkflowTags() throws Exception {
    String appName = WorkflowAppWithFork.class.getSimpleName();
    ApplicationId appId = NamespaceId.DEFAULT.app(appName);
    ApplicationClass applicationClass = new ApplicationClass(WorkflowAppWithFork.class.getName(), appName, null);
    String workflowName = WorkflowAppWithFork.WorkflowWithFork.class.getSimpleName();
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact(appId.getApplication(), "1.0");
    ApplicationWithPrograms appWithPrograms = createAppWithWorkflow(artifactId, appId, workflowName, applicationClass);
    WorkflowSpecification workflowSpec = appWithPrograms.getSpecification().getWorkflows().get(workflowName);
    MetadataWriterStage systemMetadataWriterStage = new MetadataWriterStage(metadataServiceClient);
    StageContext stageContext = new StageContext(Object.class);
    systemMetadataWriterStage.process(stageContext);
    systemMetadataWriterStage.process(appWithPrograms);
    Assert.assertEquals(false, metadataStorage.read(new Read(appId.workflow(workflowName).toMetadataEntity(), MetadataScope.SYSTEM, MetadataKind.TAG)).isEmpty());
    Set<String> workflowSystemTags = metadataStorage.read(new Read(appId.workflow(workflowName).toMetadataEntity())).getTags(MetadataScope.SYSTEM);
    Sets.SetView<String> intersection = Sets.intersection(workflowSystemTags, getWorkflowForkNodes(workflowSpec));
    Assert.assertTrue("Workflows should not be tagged with fork node names, but found the following fork nodes " + "in the workflow's system tags: " + intersection, intersection.isEmpty());
    Assert.assertEquals(false, metadataStorage.read(new Read(appId.toMetadataEntity(), MetadataScope.SYSTEM, MetadataKind.PROPERTY)).isEmpty());
    Map<String, String> metadataProperties = metadataStorage.read(new Read(appId.toMetadataEntity())).getProperties(MetadataScope.SYSTEM);
    Assert.assertEquals(WorkflowAppWithFork.SCHED_NAME + ":testDescription", metadataProperties.get("schedule:" + WorkflowAppWithFork.SCHED_NAME));
}

Example 18 with WorkflowSpecification

use of io.cdap.cdap.api.workflow.WorkflowSpecification in project cdap by cdapio.

the class ApplicationVerificationStage method verifyPrograms.

protected void verifyPrograms(ApplicationId appId, ApplicationSpecification specification) {
    Iterable<ProgramSpecification> programSpecs = Iterables.concat(specification.getMapReduce().values(), specification.getWorkflows().values());
    VerifyResult result;
    for (ProgramSpecification programSpec : programSpecs) {
        Verifier<ProgramSpecification> verifier = getVerifier(programSpec.getClass());
        result = verifier.verify(appId, programSpec);
        if (!result.isSuccess()) {
            throw new RuntimeException(result.getMessage());
        }
    }
    for (Map.Entry<String, WorkflowSpecification> entry : specification.getWorkflows().entrySet()) {
        verifyWorkflowSpecifications(specification, entry.getValue());
    }
    for (Map.Entry<String, ScheduleCreationSpec> entry : specification.getProgramSchedules().entrySet()) {
        String programName = entry.getValue().getProgramName();
        if (!specification.getWorkflows().containsKey(programName)) {
            throw new RuntimeException(String.format("Schedule '%s' is invalid: Workflow '%s' is not configured " + "in application '%s'", entry.getValue().getName(), programName, specification.getName()));
        }
    }
}

Example 19 with WorkflowSpecification

use of io.cdap.cdap.api.workflow.WorkflowSpecification in project cdap by cdapio.

the class WorkflowHttpHandler method getWorkflowSpecForValidRun.

/**
 * Get the {@link WorkflowSpecification} if valid application id, workflow id, and runid are provided.
 * @param namespaceId the namespace id
 * @param applicationId the application id
 * @param workflowId the workflow id
 * @param runId the runid of the workflow
 * @return the specifications for the Workflow
 * @throws NotFoundException is thrown when the application, workflow, or runid is not found
 */
private WorkflowSpecification getWorkflowSpecForValidRun(String namespaceId, String applicationId, String workflowId, String runId) throws NotFoundException {
    ApplicationId appId = new ApplicationId(namespaceId, applicationId);
    ApplicationSpecification appSpec = store.getApplication(appId);
    if (appSpec == null) {
        throw new ApplicationNotFoundException(appId);
    }
    WorkflowSpecification workflowSpec = appSpec.getWorkflows().get(workflowId);
    ProgramId programId = new ProgramId(namespaceId, applicationId, ProgramType.WORKFLOW, workflowId);
    if (workflowSpec == null) {
        throw new ProgramNotFoundException(programId);
    }
    if (store.getRun(programId.run(runId)) == null) {
        throw new NotFoundException(new ProgramRunId(programId.getNamespace(), programId.getApplication(), programId.getType(), programId.getProgram(), runId));
    }
    return workflowSpec;
}

Example 20 with WorkflowSpecification

use of io.cdap.cdap.api.workflow.WorkflowSpecification in project cdap by cdapio.

the class DefaultStore method recordCompletedWorkflow.

private void recordCompletedWorkflow(AppMetadataStore metaStore, WorkflowTable workflowTable, WorkflowId workflowId, String runId) throws IOException, TableNotFoundException {
    RunRecordDetail runRecord = metaStore.getRun(workflowId.run(runId));
    if (runRecord == null) {
        return;
    }
    ApplicationId app = workflowId.getParent();
    ApplicationSpecification appSpec = getApplicationSpec(metaStore, app);
    if (appSpec == null || appSpec.getWorkflows() == null || appSpec.getWorkflows().get(workflowId.getProgram()) == null) {
        LOG.warn("Missing ApplicationSpecification for {}, " + "potentially caused by application removal right after stopping workflow {}", app, workflowId);
        return;
    }
    boolean workFlowNodeFailed = false;
    WorkflowSpecification workflowSpec = appSpec.getWorkflows().get(workflowId.getProgram());
    Map<String, WorkflowNode> nodeIdMap = workflowSpec.getNodeIdMap();
    List<WorkflowTable.ProgramRun> programRunsList = new ArrayList<>();
    for (Map.Entry<String, String> entry : runRecord.getProperties().entrySet()) {
        if (!("workflowToken".equals(entry.getKey()) || "runtimeArgs".equals(entry.getKey()) || "workflowNodeState".equals(entry.getKey()))) {
            WorkflowActionNode workflowNode = (WorkflowActionNode) nodeIdMap.get(entry.getKey());
            ProgramType programType = ProgramType.valueOfSchedulableType(workflowNode.getProgram().getProgramType());
            ProgramId innerProgram = app.program(programType, entry.getKey());
            RunRecordDetail innerProgramRun = metaStore.getRun(innerProgram.run(entry.getValue()));
            if (innerProgramRun != null && innerProgramRun.getStatus().equals(ProgramRunStatus.COMPLETED)) {
                Long stopTs = innerProgramRun.getStopTs();
                // since the program is completed, the stop ts cannot be null
                if (stopTs == null) {
                    LOG.warn("Since the program has completed, expected its stop time to not be null. " + "Not writing workflow completed record for Program = {}, Workflow = {}, Run = {}", innerProgram, workflowId, runRecord);
                    workFlowNodeFailed = true;
                    break;
                }
                programRunsList.add(new WorkflowTable.ProgramRun(entry.getKey(), entry.getValue(), programType, stopTs - innerProgramRun.getStartTs()));
            } else {
                workFlowNodeFailed = true;
                break;
            }
        }
    }
    if (workFlowNodeFailed) {
        return;
    }
    workflowTable.write(workflowId, runRecord, programRunsList);
}