Examples with WorkflowNodeStateDetail io.cdap.cdap.proto.WorkflowNodeStateDetail used on opensource projects

Example 6 with WorkflowNodeStateDetail

use of io.cdap.cdap.proto.WorkflowNodeStateDetail in project cdap by cdapio.

the class ProgramNotificationSubscriberServiceTest method testWorkflowInnerPrograms.

@Test
public void testWorkflowInnerPrograms() throws Exception {
    AppFabricTestHelper.deployApplication(Id.Namespace.DEFAULT, ProgramStateWorkflowApp.class, null, cConf);
    ProgramRunId workflowRunId = NamespaceId.DEFAULT.app(ProgramStateWorkflowApp.class.getSimpleName()).workflow(ProgramStateWorkflowApp.ProgramStateWorkflow.class.getSimpleName()).run(RunIds.generate());
    ApplicationSpecification appSpec = TransactionRunners.run(transactionRunner, context -> {
        return AppMetadataStore.create(context).getApplication(workflowRunId.getParent().getParent()).getSpec();
    });
    ProgramDescriptor programDescriptor = new ProgramDescriptor(workflowRunId.getParent(), appSpec);
    // Start and run the workflow
    Map<String, String> systemArgs = new HashMap<>();
    systemArgs.put(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString());
    systemArgs.put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName());
    programStateWriter.start(workflowRunId, new SimpleProgramOptions(workflowRunId.getParent(), new BasicArguments(systemArgs), new BasicArguments()), null, programDescriptor);
    programStateWriter.running(workflowRunId, null);
    ProgramRunId mrRunId = workflowRunId.getParent().getParent().mr(ProgramStateWorkflowApp.ProgramStateMR.class.getSimpleName()).run(RunIds.generate());
    ProgramRunId sparkRunId = workflowRunId.getParent().getParent().spark(ProgramStateWorkflowApp.ProgramStateSpark.class.getSimpleName()).run(RunIds.generate());
    ProgramId sparkId2 = workflowRunId.getParent().getParent().spark(ProgramStateWorkflowApp.ProgramStateSpark2.class.getSimpleName());
    // Start and run the MR and Spark inside
    for (ProgramRunId programRunId : Arrays.asList(mrRunId, sparkRunId)) {
        workflowStateWriter.addWorkflowNodeState(workflowRunId, new WorkflowNodeStateDetail(programRunId.getProgram(), NodeStatus.STARTING));
        workflowStateWriter.addWorkflowNodeState(workflowRunId, new WorkflowNodeStateDetail(programRunId.getProgram(), NodeStatus.RUNNING));
        systemArgs = new HashMap<>(systemArgs);
        systemArgs.put(ProgramOptionConstants.RUN_ID, programRunId.getRun());
        systemArgs.put(ProgramOptionConstants.WORKFLOW_NAME, workflowRunId.getProgram());
        systemArgs.put(ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId.getRun());
        systemArgs.put(ProgramOptionConstants.WORKFLOW_NODE_ID, programRunId.getProgram());
        systemArgs.put(ProgramOptionConstants.PROGRAM_NAME_IN_WORKFLOW, programRunId.getProgram());
        programStateWriter.start(programRunId, new SimpleProgramOptions(programRunId.getParent(), new BasicArguments(systemArgs), new BasicArguments()), null, programDescriptor);
        programStateWriter.running(programRunId, null);
        // Wait for the inner program running
        Tasks.waitFor(ProgramRunStatus.RUNNING, () -> TransactionRunners.run(transactionRunner, context -> {
            AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
            RunRecordDetail meta = metadataStoreDataset.getRun(programRunId);
            if (meta == null) {
                return null;
            }
            return meta.getStatus();
        }), 10, TimeUnit.SECONDS);
    }
    // Stop the Spark normally
    programStateWriter.completed(sparkRunId);
    // Error out the Workflow without stopping the MR
    programStateWriter.error(workflowRunId, new IllegalStateException("Explicitly error out"));
    // Wait for the Workflow state changed to failed
    Tasks.waitFor(ProgramRunStatus.FAILED, () -> TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
        RunRecordDetail meta = metadataStoreDataset.getRun(workflowRunId);
        if (meta == null) {
            return null;
        }
        return meta.getStatus();
    }), 10000, TimeUnit.SECONDS);
    // The MR run record should be changed to ERROR state as well (without race)
    TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
        RunRecordDetail meta = metadataStoreDataset.getRun(mrRunId);
        Assert.assertNotNull(meta);
        Assert.assertEquals(ProgramRunStatus.FAILED, meta.getStatus());
    });
    // The Spark run record should stay as COMPLETED
    TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
        RunRecordDetail meta = metadataStoreDataset.getRun(sparkRunId);
        Assert.assertNotNull(meta);
        Assert.assertEquals(ProgramRunStatus.COMPLETED, meta.getStatus());
    });
    // Since the Spark2 program hasn't been executed, there should be no run record
    TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
        Map<ProgramRunId, RunRecordDetail> runs = metadataStoreDataset.getRuns(sparkId2, ProgramRunStatus.ALL, 0, Long.MAX_VALUE, 100, null);
        Assert.assertTrue(runs.isEmpty());
    });
}

Example 7 with WorkflowNodeStateDetail

use of io.cdap.cdap.proto.WorkflowNodeStateDetail in project cdap by cdapio.

the class TestFrameworkTestRun method executeWorkflow.

private String executeWorkflow(ApplicationManager applicationManager, Map<String, String> additionalParams, int expectedComplete) throws Exception {
    WorkflowManager wfManager = applicationManager.getWorkflowManager(WorkflowAppWithLocalDatasets.WORKFLOW_NAME);
    Map<String, String> runtimeArgs = new HashMap<>();
    File waitFile = new File(TMP_FOLDER.newFolder(), "/wait.file");
    File doneFile = new File(TMP_FOLDER.newFolder(), "/done.file");
    runtimeArgs.put("input.path", "input");
    runtimeArgs.put("output.path", "output");
    runtimeArgs.put("wait.file", waitFile.getAbsolutePath());
    runtimeArgs.put("done.file", doneFile.getAbsolutePath());
    runtimeArgs.putAll(additionalParams);
    wfManager.start(runtimeArgs);
    // Wait until custom action in the Workflow is triggered.
    while (!waitFile.exists()) {
        TimeUnit.MILLISECONDS.sleep(50);
    }
    // Now the Workflow should have RUNNING status. Get its runid.
    List<RunRecord> history = wfManager.getHistory(ProgramRunStatus.RUNNING);
    Assert.assertEquals(1, history.size());
    String runId = history.get(0).getPid();
    // Get the local datasets for this Workflow run
    DataSetManager<KeyValueTable> localDataset = getDataset(testSpace.dataset(WorkflowAppWithLocalDatasets.WORDCOUNT_DATASET + "." + runId));
    Assert.assertEquals("2", Bytes.toString(localDataset.get().read("text")));
    DataSetManager<FileSet> fileSetDataset = getDataset(testSpace.dataset(WorkflowAppWithLocalDatasets.CSV_FILESET_DATASET + "." + runId));
    Assert.assertNotNull(fileSetDataset.get());
    // Local datasets should not exist at the namespace level
    localDataset = getDataset(testSpace.dataset(WorkflowAppWithLocalDatasets.WORDCOUNT_DATASET));
    Assert.assertNull(localDataset.get());
    fileSetDataset = getDataset(testSpace.dataset(WorkflowAppWithLocalDatasets.CSV_FILESET_DATASET));
    Assert.assertNull(fileSetDataset.get());
    // Verify that the workflow hasn't completed on its own before we signal it to
    history = wfManager.getHistory(ProgramRunStatus.RUNNING);
    Assert.assertEquals(1, history.size());
    // Signal the Workflow to continue
    doneFile.createNewFile();
    // Wait for workflow to finish
    wfManager.waitForRuns(ProgramRunStatus.COMPLETED, expectedComplete, 1, TimeUnit.MINUTES);
    Map<String, WorkflowNodeStateDetail> nodeStateDetailMap = wfManager.getWorkflowNodeStates(runId);
    Map<String, String> workflowMetricsContext = new HashMap<>();
    workflowMetricsContext.put(Constants.Metrics.Tag.NAMESPACE, testSpace.getNamespace());
    workflowMetricsContext.put(Constants.Metrics.Tag.APP, applicationManager.getInfo().getName());
    workflowMetricsContext.put(Constants.Metrics.Tag.WORKFLOW, WorkflowAppWithLocalDatasets.WORKFLOW_NAME);
    workflowMetricsContext.put(Constants.Metrics.Tag.RUN_ID, runId);
    Map<String, String> writerContext = new HashMap<>(workflowMetricsContext);
    writerContext.put(Constants.Metrics.Tag.NODE, WorkflowAppWithLocalDatasets.LocalDatasetWriter.class.getSimpleName());
    Assert.assertEquals(2, getMetricsManager().getTotalMetric(writerContext, "user.num.lines"));
    Map<String, String> wfSparkMetricsContext = new HashMap<>(workflowMetricsContext);
    wfSparkMetricsContext.put(Constants.Metrics.Tag.NODE, "JavaSparkCSVToSpaceConverter");
    Assert.assertEquals(2, getMetricsManager().getTotalMetric(wfSparkMetricsContext, "user.num.lines"));
    // check in spark context
    Map<String, String> sparkMetricsContext = new HashMap<>();
    sparkMetricsContext.put(Constants.Metrics.Tag.NAMESPACE, testSpace.getNamespace());
    sparkMetricsContext.put(Constants.Metrics.Tag.APP, applicationManager.getInfo().getName());
    sparkMetricsContext.put(Constants.Metrics.Tag.SPARK, "JavaSparkCSVToSpaceConverter");
    sparkMetricsContext.put(Constants.Metrics.Tag.RUN_ID, nodeStateDetailMap.get("JavaSparkCSVToSpaceConverter").getRunId());
    Assert.assertEquals(2, getMetricsManager().getTotalMetric(sparkMetricsContext, "user.num.lines"));
    Map<String, String> appMetricsContext = new HashMap<>();
    appMetricsContext.put(Constants.Metrics.Tag.NAMESPACE, testSpace.getNamespace());
    appMetricsContext.put(Constants.Metrics.Tag.APP, applicationManager.getInfo().getName());
    // app metrics context should have sum from custom action and spark metrics.
    Assert.assertEquals(4, getMetricsManager().getTotalMetric(appMetricsContext, "user.num.lines"));
    Map<String, String> wfMRMetricsContext = new HashMap<>(workflowMetricsContext);
    wfMRMetricsContext.put(Constants.Metrics.Tag.NODE, "WordCount");
    Assert.assertEquals(7, getMetricsManager().getTotalMetric(wfMRMetricsContext, "user.num.words"));
    // mr metrics context
    Map<String, String> mrMetricsContext = new HashMap<>();
    mrMetricsContext.put(Constants.Metrics.Tag.NAMESPACE, testSpace.getNamespace());
    mrMetricsContext.put(Constants.Metrics.Tag.APP, applicationManager.getInfo().getName());
    mrMetricsContext.put(Constants.Metrics.Tag.MAPREDUCE, "WordCount");
    mrMetricsContext.put(Constants.Metrics.Tag.RUN_ID, nodeStateDetailMap.get("WordCount").getRunId());
    Assert.assertEquals(7, getMetricsManager().getTotalMetric(mrMetricsContext, "user.num.words"));
    final Map<String, String> readerContext = new HashMap<>(workflowMetricsContext);
    readerContext.put(Constants.Metrics.Tag.NODE, "readerAction");
    Tasks.waitFor(6L, new Callable<Long>() {

        @Override
        public Long call() throws Exception {
            return getMetricsManager().getTotalMetric(readerContext, "user.unique.words");
        }
    }, 60, TimeUnit.SECONDS);
    return runId;
}

Example 8 with WorkflowNodeStateDetail

use of io.cdap.cdap.proto.WorkflowNodeStateDetail in project cdap by caskdata.

the class WorkflowDriver method executeCustomAction.

private void executeCustomAction(final WorkflowActionNode node, InstantiatorFactory instantiator, final ClassLoader classLoader, WorkflowToken token) throws Exception {
    CustomActionExecutor customActionExecutor;
    // Node has CustomActionSpecification, so it must represent the CustomAction added in 3.5.0
    // Create instance of the CustomActionExecutor using CustomActionContext
    WorkflowProgramInfo info = new WorkflowProgramInfo(workflowSpec.getName(), node.getNodeId(), workflowRunId.getRun(), node.getNodeId(), (BasicWorkflowToken) token, workflowContext.fieldLineageConsolidationEnabled());
    ProgramOptions actionOptions = new SimpleProgramOptions(programOptions.getProgramId(), programOptions.getArguments(), new BasicArguments(RuntimeArguments.extractScope(ACTION_SCOPE, node.getNodeId(), programOptions.getUserArguments().asMap())));
    BasicCustomActionContext context = new BasicCustomActionContext(program, actionOptions, cConf, node.getCustomActionSpecification(), info, metricsCollectionService, datasetFramework, txClient, discoveryServiceClient, pluginInstantiator, secureStore, secureStoreManager, messagingService, metadataReader, metadataPublisher, namespaceQueryAdmin, fieldLineageWriter, remoteClientFactory);
    customActionExecutor = new CustomActionExecutor(context, instantiator, classLoader);
    status.put(node.getNodeId(), node);
    workflowStateWriter.addWorkflowNodeState(workflowRunId, new WorkflowNodeStateDetail(node.getNodeId(), NodeStatus.RUNNING));
    Throwable failureCause = null;
    try {
        customActionExecutor.execute();
    } catch (Throwable t) {
        failureCause = t;
        throw t;
    } finally {
        status.remove(node.getNodeId());
        workflowStateWriter.setWorkflowToken(workflowRunId, token);
        NodeStatus status = failureCause == null ? NodeStatus.COMPLETED : NodeStatus.FAILED;
        if (failureCause == null) {
            writeFieldLineage(context);
        }
        nodeStates.put(node.getNodeId(), new WorkflowNodeState(node.getNodeId(), status, null, failureCause));
        BasicThrowable defaultThrowable = failureCause == null ? null : new BasicThrowable(failureCause);
        workflowStateWriter.addWorkflowNodeState(workflowRunId, new WorkflowNodeStateDetail(node.getNodeId(), status, null, defaultThrowable));
    }
}

Example 9 with WorkflowNodeStateDetail

use of io.cdap.cdap.proto.WorkflowNodeStateDetail in project cdap by caskdata.

the class AppFabricClient method getWorkflowNodeStates.

public Map<String, WorkflowNodeStateDetail> getWorkflowNodeStates(ProgramRunId workflowRunId) throws NotFoundException, UnauthorizedException {
    MockResponder responder = new MockResponder();
    String uri = String.format("%s/apps/%s/workflows/%s/runs/%s/nodes/state", getNamespacePath(workflowRunId.getNamespace()), workflowRunId.getApplication(), workflowRunId.getProgram(), workflowRunId.getRun());
    HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, uri);
    workflowHttpHandler.getWorkflowNodeStates(request, responder, workflowRunId.getNamespace(), workflowRunId.getApplication(), workflowRunId.getProgram(), workflowRunId.getRun());
    Type nodeStatesType = new TypeToken<Map<String, WorkflowNodeStateDetail>>() {
    }.getType();
    Map<String, WorkflowNodeStateDetail> nodeStates = responder.decodeResponseContent(nodeStatesType, GSON);
    verifyResponse(HttpResponseStatus.OK, responder.getStatus(), "Getting workflow node states failed.");
    return nodeStates;
}

Example 10 with WorkflowNodeStateDetail

use of io.cdap.cdap.proto.WorkflowNodeStateDetail in project cdap by caskdata.

the class WorkflowHttpHandlerTest method verifyMultipleConcurrentRuns.

private void verifyMultipleConcurrentRuns(Id.Program workflowId) throws Exception {
    verifyProgramRuns(workflowId, ProgramRunStatus.RUNNING, 1);
    List<RunRecord> historyRuns = getProgramRuns(workflowId, ProgramRunStatus.RUNNING);
    Assert.assertEquals(2, historyRuns.size());
    HttpResponse response = getWorkflowNodeStatesResponse(workflowId.toEntityId(), historyRuns.get(0).getPid());
    Assert.assertEquals(200, response.getResponseCode());
    Map<String, WorkflowNodeStateDetail> statusMap = readResponse(response, MAP_STRING_TO_WORKFLOWNODESTATEDETAIL_TYPE);
    Assert.assertEquals(1, statusMap.size());
    Assert.assertEquals(ConcurrentWorkflowApp.SimpleAction.class.getSimpleName(), statusMap.values().iterator().next().getNodeId());
    response = getWorkflowNodeStatesResponse(workflowId.toEntityId(), historyRuns.get(1).getPid());
    Assert.assertEquals(200, response.getResponseCode());
    statusMap = readResponse(response, MAP_STRING_TO_WORKFLOWNODESTATEDETAIL_TYPE);
    Assert.assertEquals(1, statusMap.size());
    Assert.assertEquals(ConcurrentWorkflowApp.SimpleAction.class.getSimpleName(), statusMap.values().iterator().next().getNodeId());
}