Examples with WorkflowNodeStateDetail co.cask.cdap.proto.WorkflowNodeStateDetail used on opensource projects

Example 1 with WorkflowNodeStateDetail

use of co.cask.cdap.proto.WorkflowNodeStateDetail in project cdap by caskdata.

the class WorkflowHttpHandlerTest method testWorkflowScopedArguments.

@Category(XSlowTests.class)
@Test
public void testWorkflowScopedArguments() throws Exception {
    String workflowRunIdProperty = "workflowrunid";
    HttpResponse response = deploy(WorkflowAppWithScopedParameters.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);
    Assert.assertEquals(200, response.getStatusLine().getStatusCode());
    ProgramId programId = Ids.namespace(TEST_NAMESPACE2).app(WorkflowAppWithScopedParameters.APP_NAME).workflow(WorkflowAppWithScopedParameters.ONE_WORKFLOW);
    Map<String, String> runtimeArguments = Maps.newHashMap();
    runtimeArguments.put("debug", "true");
    runtimeArguments.put("mapreduce.*.debug", "false");
    runtimeArguments.put("mapreduce.OneMR.debug", "true");
    runtimeArguments.put("input.path", createInput("ProgramInput"));
    runtimeArguments.put("mapreduce.OneMR.input.path", createInput("OneMRInput"));
    runtimeArguments.put("mapreduce.OneMR.logical.start.time", "1234567890000");
    runtimeArguments.put("mapreduce.AnotherMR.input.path", createInput("AnotherMRInput"));
    runtimeArguments.put("spark.*.input.path", createInput("SparkInput"));
    runtimeArguments.put("output.path", new File(tmpFolder.newFolder(), "ProgramOutput").getAbsolutePath());
    runtimeArguments.put("mapreduce.OneMR.output.path", new File(tmpFolder.newFolder(), "OneMROutput").getAbsolutePath());
    runtimeArguments.put("spark.AnotherSpark.output.path", new File(tmpFolder.newFolder(), "AnotherSparkOutput").getAbsolutePath());
    runtimeArguments.put("mapreduce.*.processing.time", "1HR");
    runtimeArguments.put("dataset.Purchase.cache.seconds", "30");
    runtimeArguments.put("dataset.UserProfile.schema.property", "constant");
    runtimeArguments.put("dataset.unknown.dataset", "false");
    runtimeArguments.put("dataset.*.read.timeout", "60");
    setAndTestRuntimeArgs(programId.toId(), runtimeArguments);
    // Start the workflow
    startProgram(programId.toId());
    waitState(programId.toId(), ProgramStatus.RUNNING.name());
    // Wait until we have a run record
    verifyProgramRuns(programId.toId(), "running");
    List<RunRecord> workflowHistoryRuns = getProgramRuns(programId.toId(), "running");
    String workflowRunId = workflowHistoryRuns.get(0).getPid();
    Id.Program mr1ProgramId = Id.Program.from(TEST_NAMESPACE2, WorkflowAppWithScopedParameters.APP_NAME, ProgramType.MAPREDUCE, WorkflowAppWithScopedParameters.ONE_MR);
    waitState(mr1ProgramId, ProgramStatus.RUNNING.name());
    List<RunRecord> oneMRHistoryRuns = getProgramRuns(mr1ProgramId, "running");
    String expectedMessage = String.format("Cannot stop the program '%s' started by the Workflow run '%s'. " + "Please stop the Workflow.", new Id.Run(mr1ProgramId, oneMRHistoryRuns.get(0).getPid()), workflowRunId);
    stopProgram(mr1ProgramId, oneMRHistoryRuns.get(0).getPid(), 400, expectedMessage);
    verifyProgramRuns(programId.toId(), "completed");
    workflowHistoryRuns = getProgramRuns(programId.toId(), "completed");
    oneMRHistoryRuns = getProgramRuns(mr1ProgramId, "completed");
    Id.Program mr2ProgramId = Id.Program.from(TEST_NAMESPACE2, WorkflowAppWithScopedParameters.APP_NAME, ProgramType.MAPREDUCE, WorkflowAppWithScopedParameters.ANOTHER_MR);
    List<RunRecord> anotherMRHistoryRuns = getProgramRuns(mr2ProgramId, "completed");
    Id.Program spark1ProgramId = Id.Program.from(TEST_NAMESPACE2, WorkflowAppWithScopedParameters.APP_NAME, ProgramType.SPARK, WorkflowAppWithScopedParameters.ONE_SPARK);
    List<RunRecord> oneSparkHistoryRuns = getProgramRuns(spark1ProgramId, "completed");
    Id.Program spark2ProgramId = Id.Program.from(TEST_NAMESPACE2, WorkflowAppWithScopedParameters.APP_NAME, ProgramType.SPARK, WorkflowAppWithScopedParameters.ANOTHER_SPARK);
    List<RunRecord> anotherSparkHistoryRuns = getProgramRuns(spark2ProgramId, "completed");
    Assert.assertEquals(1, workflowHistoryRuns.size());
    Assert.assertEquals(1, oneMRHistoryRuns.size());
    Assert.assertEquals(1, anotherMRHistoryRuns.size());
    Assert.assertEquals(1, oneSparkHistoryRuns.size());
    Assert.assertEquals(1, anotherSparkHistoryRuns.size());
    Map<String, String> workflowRunRecordProperties = workflowHistoryRuns.get(0).getProperties();
    Map<String, String> oneMRRunRecordProperties = oneMRHistoryRuns.get(0).getProperties();
    Map<String, String> anotherMRRunRecordProperties = anotherMRHistoryRuns.get(0).getProperties();
    Map<String, String> oneSparkRunRecordProperties = oneSparkHistoryRuns.get(0).getProperties();
    Map<String, String> anotherSparkRunRecordProperties = anotherSparkHistoryRuns.get(0).getProperties();
    Assert.assertNotNull(oneMRRunRecordProperties.get(workflowRunIdProperty));
    Assert.assertEquals(workflowHistoryRuns.get(0).getPid(), oneMRRunRecordProperties.get(workflowRunIdProperty));
    Assert.assertNotNull(anotherMRRunRecordProperties.get(workflowRunIdProperty));
    Assert.assertEquals(workflowHistoryRuns.get(0).getPid(), anotherMRRunRecordProperties.get(workflowRunIdProperty));
    Assert.assertNotNull(oneSparkRunRecordProperties.get(workflowRunIdProperty));
    Assert.assertEquals(workflowHistoryRuns.get(0).getPid(), oneSparkRunRecordProperties.get(workflowRunIdProperty));
    Assert.assertNotNull(anotherSparkRunRecordProperties.get(workflowRunIdProperty));
    Assert.assertEquals(workflowHistoryRuns.get(0).getPid(), anotherSparkRunRecordProperties.get(workflowRunIdProperty));
    Assert.assertEquals(workflowRunRecordProperties.get(WorkflowAppWithScopedParameters.ONE_MR), oneMRHistoryRuns.get(0).getPid());
    Assert.assertEquals(workflowRunRecordProperties.get(WorkflowAppWithScopedParameters.ONE_SPARK), oneSparkHistoryRuns.get(0).getPid());
    Assert.assertEquals(workflowRunRecordProperties.get(WorkflowAppWithScopedParameters.ANOTHER_MR), anotherMRHistoryRuns.get(0).getPid());
    Assert.assertEquals(workflowRunRecordProperties.get(WorkflowAppWithScopedParameters.ANOTHER_SPARK), anotherSparkHistoryRuns.get(0).getPid());
    // Get Workflow node states
    Map<String, WorkflowNodeStateDetail> nodeStates = getWorkflowNodeStates(programId, workflowHistoryRuns.get(0).getPid());
    Assert.assertNotNull(nodeStates);
    Assert.assertEquals(5, nodeStates.size());
    WorkflowNodeStateDetail mrNodeState = nodeStates.get(WorkflowAppWithScopedParameters.ONE_MR);
    Assert.assertNotNull(mrNodeState);
    Assert.assertEquals(WorkflowAppWithScopedParameters.ONE_MR, mrNodeState.getNodeId());
    Assert.assertEquals(oneMRHistoryRuns.get(0).getPid(), mrNodeState.getRunId());
    mrNodeState = nodeStates.get(WorkflowAppWithScopedParameters.ANOTHER_MR);
    Assert.assertNotNull(mrNodeState);
    Assert.assertEquals(WorkflowAppWithScopedParameters.ANOTHER_MR, mrNodeState.getNodeId());
    Assert.assertEquals(anotherMRHistoryRuns.get(0).getPid(), mrNodeState.getRunId());
    WorkflowNodeStateDetail sparkNodeState = nodeStates.get(WorkflowAppWithScopedParameters.ONE_SPARK);
    Assert.assertNotNull(sparkNodeState);
    Assert.assertEquals(WorkflowAppWithScopedParameters.ONE_SPARK, sparkNodeState.getNodeId());
    Assert.assertEquals(oneSparkHistoryRuns.get(0).getPid(), sparkNodeState.getRunId());
    sparkNodeState = nodeStates.get(WorkflowAppWithScopedParameters.ANOTHER_SPARK);
    Assert.assertNotNull(sparkNodeState);
    Assert.assertEquals(WorkflowAppWithScopedParameters.ANOTHER_SPARK, sparkNodeState.getNodeId());
    Assert.assertEquals(anotherSparkHistoryRuns.get(0).getPid(), sparkNodeState.getRunId());
    WorkflowNodeStateDetail oneActionNodeState = nodeStates.get(WorkflowAppWithScopedParameters.ONE_ACTION);
    Assert.assertNotNull(oneActionNodeState);
    Assert.assertEquals(WorkflowAppWithScopedParameters.ONE_ACTION, oneActionNodeState.getNodeId());
}

Example 2 with WorkflowNodeStateDetail

use of co.cask.cdap.proto.WorkflowNodeStateDetail in project cdap by caskdata.

the class RemoteRuntimeStoreHandler method addWorkflowNodeState.

@POST
@Path("/addWorkflowNodeState")
public void addWorkflowNodeState(HttpRequest request, HttpResponder responder) throws Exception {
    Iterator<MethodArgument> arguments = parseArguments(request);
    ProgramRunId workflowRunId = deserializeNext(arguments);
    WorkflowNodeStateDetail nodeStateDetail = deserializeNext(arguments);
    store.addWorkflowNodeState(workflowRunId, nodeStateDetail);
    responder.sendStatus(HttpResponseStatus.OK);
}

Example 3 with WorkflowNodeStateDetail

use of co.cask.cdap.proto.WorkflowNodeStateDetail in project cdap by caskdata.

the class WorkflowDriver method executeCustomAction.

private void executeCustomAction(final WorkflowActionNode node, InstantiatorFactory instantiator, final ClassLoader classLoader, WorkflowToken token) throws Exception {
    CustomActionExecutor customActionExecutor;
    if (node.getActionSpecification() != null) {
        // Node has WorkflowActionSpecification, so it must represent the deprecated WorkflowAction
        // Create instance of the CustomActionExecutor using BasicWorkflowContext
        BasicWorkflowContext context = createWorkflowContext(node.getActionSpecification(), token);
        customActionExecutor = new CustomActionExecutor(workflowRunId, context, instantiator, classLoader);
    } else {
        // Node has CustomActionSpecification, so it must represent the CustomAction added in 3.5.0
        // Create instance of the CustomActionExecutor using CustomActionContext
        WorkflowProgramInfo info = new WorkflowProgramInfo(workflowSpec.getName(), node.getNodeId(), workflowRunId.getRun(), node.getNodeId(), (BasicWorkflowToken) token);
        ProgramOptions actionOptions = new SimpleProgramOptions(programOptions.getName(), programOptions.getArguments(), new BasicArguments(RuntimeArguments.extractScope(ACTION_SCOPE, node.getNodeId(), programOptions.getUserArguments().asMap())));
        BasicCustomActionContext context = new BasicCustomActionContext(program, actionOptions, cConf, node.getCustomActionSpecification(), info, metricsCollectionService, datasetFramework, txClient, discoveryServiceClient, pluginInstantiator, secureStore, secureStoreManager, messagingService);
        customActionExecutor = new CustomActionExecutor(workflowRunId, context, instantiator, classLoader);
    }
    status.put(node.getNodeId(), node);
    runtimeStore.addWorkflowNodeState(workflowRunId, new WorkflowNodeStateDetail(node.getNodeId(), NodeStatus.RUNNING));
    Throwable failureCause = null;
    try {
        customActionExecutor.execute();
    } catch (Throwable t) {
        failureCause = t;
        throw t;
    } finally {
        status.remove(node.getNodeId());
        runtimeStore.updateWorkflowToken(workflowRunId, token);
        NodeStatus status = failureCause == null ? NodeStatus.COMPLETED : NodeStatus.FAILED;
        nodeStates.put(node.getNodeId(), new WorkflowNodeState(node.getNodeId(), status, null, failureCause));
        BasicThrowable defaultThrowable = failureCause == null ? null : new BasicThrowable(failureCause);
        runtimeStore.addWorkflowNodeState(workflowRunId, new WorkflowNodeStateDetail(node.getNodeId(), status, null, defaultThrowable));
    }
}

Example 4 with WorkflowNodeStateDetail

use of co.cask.cdap.proto.WorkflowNodeStateDetail in project cdap by caskdata.

the class WorkflowClient method getWorkflowNodeStates.

/**
   * Get node states associated with the Workflow run.
   * @param workflowRunId run id for the Workflow
   * @return the map of node id to the {@link WorkflowNodeStateDetail}
   * @throws IOException if the error occurred during executing the http request
   * @throws UnauthenticatedException if the request is not authorized successfully in the gateway server
   * @throws NotFoundException if the workflow with given runid not found
   */
public Map<String, WorkflowNodeStateDetail> getWorkflowNodeStates(ProgramRunId workflowRunId) throws IOException, UnauthenticatedException, NotFoundException, UnauthorizedException {
    String path = String.format("apps/%s/workflows/%s/runs/%s/nodes/state", workflowRunId.getApplication(), workflowRunId.getProgram(), workflowRunId.getRun());
    NamespaceId namespaceId = workflowRunId.getNamespaceId();
    URL urlPath = config.resolveNamespacedURLV3(namespaceId, path);
    HttpResponse response = restClient.execute(HttpMethod.GET, urlPath, config.getAccessToken(), HttpURLConnection.HTTP_NOT_FOUND);
    if (response.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
        throw new NotFoundException(workflowRunId);
    }
    return ObjectResponse.fromJsonBody(response, new TypeToken<Map<String, WorkflowNodeStateDetail>>() {
    }).getResponseObject();
}

Example 5 with WorkflowNodeStateDetail

use of co.cask.cdap.proto.WorkflowNodeStateDetail in project cdap by caskdata.

the class AppMetadataStore method getWorkflowNodeStates.

/**
   * Return the {@link List} of {@link WorkflowNodeStateDetail} for a given Workflow run.
   */
public List<WorkflowNodeStateDetail> getWorkflowNodeStates(ProgramRunId workflowRunId) {
    MDSKey key = getProgramKeyBuilder(TYPE_WORKFLOW_NODE_STATE, workflowRunId.getParent()).add(workflowRunId.getRun()).build();
    List<WorkflowNodeStateDetail> nodeStateDetails = list(key, WorkflowNodeStateDetail.class);
    // one specific run-id either in the old format or in the new format.
    if (!upgradeComplete.get() && nodeStateDetails.isEmpty() && workflowRunId.getVersion().equals(ApplicationId.DEFAULT_VERSION)) {
        key = getVersionLessProgramKeyBuilder(TYPE_WORKFLOW_NODE_STATE, workflowRunId.getParent()).add(workflowRunId.getRun()).build();
        nodeStateDetails = list(key, WorkflowNodeStateDetail.class);
    }
    return nodeStateDetails;
}