Examples with WorkflowId io.cdap.cdap.proto.id.WorkflowId used on opensource projects

Example 6 with WorkflowId

use of io.cdap.cdap.proto.id.WorkflowId in project cdap by caskdata.

the class WorkflowStatsSLAHttpHandler method compare.

/**
 * Compare the metrics of 2 runs of a workflow
 *
 * @param request The request
 * @param responder The responder
 * @param namespaceId The namespace the application is in
 * @param appId The application the workflow is in
 * @param workflowId The workflow that needs to have it stats shown
 * @param runId The run id of the Workflow that the user wants to see
 * @param otherRunId The other run id of the same workflow that the user wants to compare against
 */
@GET
@Path("apps/{app-id}/workflows/{workflow-id}/runs/{run-id}/compare")
public void compare(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("app-id") String appId, @PathParam("workflow-id") String workflowId, @PathParam("run-id") String runId, @QueryParam("other-run-id") String otherRunId) throws Exception {
    WorkflowId workflow = new WorkflowId(namespaceId, appId, workflowId);
    WorkflowRunMetrics detailedStatistics = getDetailedRecord(workflow, runId);
    WorkflowRunMetrics otherDetailedStatistics = getDetailedRecord(workflow, otherRunId);
    if (detailedStatistics == null) {
        throw new NotFoundException("The run-id provided was not found : " + runId);
    }
    if (otherDetailedStatistics == null) {
        throw new NotFoundException("The other run-id provided was not found : " + otherRunId);
    }
    List<WorkflowRunMetrics> workflowRunMetricsList = new ArrayList<>();
    workflowRunMetricsList.add(detailedStatistics);
    workflowRunMetricsList.add(otherDetailedStatistics);
    responder.sendJson(HttpResponseStatus.OK, GSON.toJson(format(workflowRunMetricsList)));
}

Example 7 with WorkflowId

use of io.cdap.cdap.proto.id.WorkflowId in project cdap by caskdata.

the class ListWorkflowSchedulesCommand method perform.

@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
    String[] programIdParts = arguments.get(ElementType.WORKFLOW.getArgumentName().toString()).split("\\.");
    if (programIdParts.length < 2) {
        throw new CommandInputError(this);
    }
    final String appId = programIdParts[0];
    String workflowName = programIdParts[1];
    WorkflowId workflowId = cliConfig.getCurrentNamespace().app(appId).workflow(workflowName);
    List<ScheduleDetail> list = scheduleClient.listSchedules(workflowId);
    Table table = Table.builder().setHeader("application", "program", "program type", "name", "description", "trigger", "timeoutMillis", "properties").setRows(list, new RowMaker<ScheduleDetail>() {

        @Override
        public List<?> makeRow(ScheduleDetail object) {
            return Lists.newArrayList(appId, object.getProgram().getProgramName(), object.getProgram().getProgramType().name(), object.getName(), object.getDescription(), object.getTrigger(), object.getTimeoutMillis(), GSON.toJson(object.getProperties()));
        }
    }).build();
    cliConfig.getTableRenderer().render(cliConfig, output, table);
}

Example 8 with WorkflowId

use of io.cdap.cdap.proto.id.WorkflowId in project cdap by caskdata.

the class WorkflowHttpHandler method getWorkflowToken.

private WorkflowToken getWorkflowToken(String namespaceId, String appName, String workflow, String runId) throws NotFoundException {
    ApplicationId appId = new ApplicationId(namespaceId, appName);
    ApplicationSpecification appSpec = store.getApplication(appId);
    if (appSpec == null) {
        throw new NotFoundException(appId);
    }
    WorkflowId workflowId = appId.workflow(workflow);
    if (!appSpec.getWorkflows().containsKey(workflow)) {
        throw new NotFoundException(workflowId);
    }
    if (store.getRun(workflowId.run(runId)) == null) {
        throw new NotFoundException(workflowId.run(runId));
    }
    return store.getWorkflowToken(workflowId, runId);
}

Example 9 with WorkflowId

use of io.cdap.cdap.proto.id.WorkflowId in project cdap by caskdata.

the class WorkflowHttpHandlerTest method testWorkflowTokenPut.

@Test
public void testWorkflowTokenPut() throws Exception {
    deploy(WorkflowTokenTestPutApp.class, 200);
    Id.Application appId = Id.Application.from(Id.Namespace.DEFAULT, WorkflowTokenTestPutApp.NAME);
    Id.Workflow workflowId = Id.Workflow.from(appId, WorkflowTokenTestPutApp.WorkflowTokenTestPut.NAME);
    Id.Program sparkId = Id.Program.from(appId, ProgramType.SPARK, WorkflowTokenTestPutApp.SparkTestApp.NAME);
    // Start program with inputPath and outputPath arguments.
    // This should succeed. The programs inside the workflow will attempt to write to the workflow token
    // from the Mapper's and Reducer's methods as well as from a Spark closure, and they will throw an exception
    // if that succeeds.
    // The MapReduce's initialize will record the workflow run id in the token, and the destroy as well
    // as the mapper and the reducer will validate that they have the same workflow run id.
    String outputPath = new File(tmpFolder.newFolder(), "output").getAbsolutePath();
    startProgram(workflowId, ImmutableMap.of("inputPath", createInputForRecordVerification("sixthInput"), "outputPath", outputPath));
    waitState(workflowId, ProgramStatus.RUNNING.name());
    waitState(workflowId, ProgramStatus.STOPPED.name());
    // validate the completed workflow run and validate that it is the same as recorded in the token
    verifyProgramRuns(workflowId, ProgramRunStatus.COMPLETED);
    List<RunRecord> runs = getProgramRuns(workflowId, ProgramRunStatus.COMPLETED);
    Assert.assertEquals(1, runs.size());
    String wfRunId = runs.get(0).getPid();
    WorkflowTokenDetail tokenDetail = getWorkflowToken(workflowId, wfRunId, null, null);
    List<WorkflowTokenDetail.NodeValueDetail> details = tokenDetail.getTokenData().get("wf.runid");
    Assert.assertEquals(1, details.size());
    Assert.assertEquals(wfRunId, details.get(0).getValue());
    // validate that none of the mapper, reducer or spark closure were able to write to the token
    for (String key : new String[] { "mapper.initialize.key", "map.key", "reducer.initialize.key", "reduce.key", "some.key" }) {
        Assert.assertFalse(tokenDetail.getTokenData().containsKey(key));
    }
    List<RunRecord> sparkProgramRuns = getProgramRuns(sparkId, ProgramRunStatus.COMPLETED);
    Assert.assertEquals(1, sparkProgramRuns.size());
}

Example 10 with WorkflowId

use of io.cdap.cdap.proto.id.WorkflowId in project cdap by caskdata.

the class WorkflowHttpHandlerTest method testWorkflowScopedArguments.

@Category(XSlowTests.class)
@Test
public void testWorkflowScopedArguments() throws Exception {
    String workflowRunIdProperty = "workflowrunid";
    deploy(WorkflowAppWithScopedParameters.class, 200, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);
    ProgramId programId = Ids.namespace(TEST_NAMESPACE2).app(WorkflowAppWithScopedParameters.APP_NAME).workflow(WorkflowAppWithScopedParameters.ONE_WORKFLOW);
    Map<String, String> runtimeArguments = Maps.newHashMap();
    runtimeArguments.put("debug", "true");
    runtimeArguments.put("mapreduce.*.debug", "false");
    runtimeArguments.put("mapreduce.OneMR.debug", "true");
    runtimeArguments.put("input.path", createInput("ProgramInput"));
    runtimeArguments.put("mapreduce.OneMR.input.path", createInput("OneMRInput"));
    runtimeArguments.put("mapreduce.OneMR.logical.start.time", "1234567890000");
    runtimeArguments.put("mapreduce.AnotherMR.input.path", createInput("AnotherMRInput"));
    runtimeArguments.put("spark.*.input.path", createInput("SparkInput"));
    runtimeArguments.put("output.path", new File(tmpFolder.newFolder(), "ProgramOutput").getAbsolutePath());
    runtimeArguments.put("mapreduce.OneMR.output.path", new File(tmpFolder.newFolder(), "OneMROutput").getAbsolutePath());
    runtimeArguments.put("spark.AnotherSpark.output.path", new File(tmpFolder.newFolder(), "AnotherSparkOutput").getAbsolutePath());
    runtimeArguments.put("mapreduce.*.processing.time", "1HR");
    runtimeArguments.put("dataset.Purchase.cache.seconds", "30");
    runtimeArguments.put("dataset.UserProfile.schema.property", "constant");
    runtimeArguments.put("dataset.unknown.dataset", "false");
    runtimeArguments.put("dataset.*.read.timeout", "60");
    setAndTestRuntimeArgs(Id.Program.fromEntityId(programId), runtimeArguments);
    // Start the workflow
    startProgram(programId, 200);
    // Wait until we have a run record
    verifyProgramRuns(programId, ProgramRunStatus.RUNNING);
    List<RunRecord> workflowHistoryRuns = getProgramRuns(programId, ProgramRunStatus.ALL);
    String workflowRunId = workflowHistoryRuns.get(0).getPid();
    ProgramId mr1ProgramId = Ids.namespace(TEST_NAMESPACE2).app(WorkflowAppWithScopedParameters.APP_NAME).mr(WorkflowAppWithScopedParameters.ONE_MR);
    verifyProgramRuns(mr1ProgramId, ProgramRunStatus.RUNNING);
    List<RunRecord> oneMRHistoryRuns = getProgramRuns(mr1ProgramId, ProgramRunStatus.ALL);
    String expectedMessage = String.format("Cannot stop the program '%s' started by the Workflow run '%s'. " + "Please stop the Workflow.", mr1ProgramId.run(oneMRHistoryRuns.get(0).getPid()), workflowRunId);
    stopProgram(Id.Program.fromEntityId(mr1ProgramId), oneMRHistoryRuns.get(0).getPid(), 400, expectedMessage);
    verifyProgramRuns(Id.Program.fromEntityId(programId), ProgramRunStatus.COMPLETED);
    workflowHistoryRuns = getProgramRuns(Id.Program.fromEntityId(programId), ProgramRunStatus.COMPLETED);
    oneMRHistoryRuns = getProgramRuns(mr1ProgramId, ProgramRunStatus.COMPLETED);
    Id.Program mr2ProgramId = Id.Program.from(TEST_NAMESPACE2, WorkflowAppWithScopedParameters.APP_NAME, ProgramType.MAPREDUCE, WorkflowAppWithScopedParameters.ANOTHER_MR);
    List<RunRecord> anotherMRHistoryRuns = getProgramRuns(mr2ProgramId, ProgramRunStatus.COMPLETED);
    Id.Program spark1ProgramId = Id.Program.from(TEST_NAMESPACE2, WorkflowAppWithScopedParameters.APP_NAME, ProgramType.SPARK, WorkflowAppWithScopedParameters.ONE_SPARK);
    List<RunRecord> oneSparkHistoryRuns = getProgramRuns(spark1ProgramId, ProgramRunStatus.COMPLETED);
    Id.Program spark2ProgramId = Id.Program.from(TEST_NAMESPACE2, WorkflowAppWithScopedParameters.APP_NAME, ProgramType.SPARK, WorkflowAppWithScopedParameters.ANOTHER_SPARK);
    List<RunRecord> anotherSparkHistoryRuns = getProgramRuns(spark2ProgramId, ProgramRunStatus.COMPLETED);
    Assert.assertEquals(1, workflowHistoryRuns.size());
    Assert.assertEquals(1, oneMRHistoryRuns.size());
    Assert.assertEquals(1, anotherMRHistoryRuns.size());
    Assert.assertEquals(1, oneSparkHistoryRuns.size());
    Assert.assertEquals(1, anotherSparkHistoryRuns.size());
    Map<String, String> workflowRunRecordProperties = workflowHistoryRuns.get(0).getProperties();
    Map<String, String> oneMRRunRecordProperties = oneMRHistoryRuns.get(0).getProperties();
    Map<String, String> anotherMRRunRecordProperties = anotherMRHistoryRuns.get(0).getProperties();
    Map<String, String> oneSparkRunRecordProperties = oneSparkHistoryRuns.get(0).getProperties();
    Map<String, String> anotherSparkRunRecordProperties = anotherSparkHistoryRuns.get(0).getProperties();
    Assert.assertNotNull(oneMRRunRecordProperties.get(workflowRunIdProperty));
    Assert.assertEquals(workflowHistoryRuns.get(0).getPid(), oneMRRunRecordProperties.get(workflowRunIdProperty));
    Assert.assertNotNull(anotherMRRunRecordProperties.get(workflowRunIdProperty));
    Assert.assertEquals(workflowHistoryRuns.get(0).getPid(), anotherMRRunRecordProperties.get(workflowRunIdProperty));
    Assert.assertNotNull(oneSparkRunRecordProperties.get(workflowRunIdProperty));
    Assert.assertEquals(workflowHistoryRuns.get(0).getPid(), oneSparkRunRecordProperties.get(workflowRunIdProperty));
    Assert.assertNotNull(anotherSparkRunRecordProperties.get(workflowRunIdProperty));
    Assert.assertEquals(workflowHistoryRuns.get(0).getPid(), anotherSparkRunRecordProperties.get(workflowRunIdProperty));
    Assert.assertEquals(workflowRunRecordProperties.get(WorkflowAppWithScopedParameters.ONE_MR), oneMRHistoryRuns.get(0).getPid());
    Assert.assertEquals(workflowRunRecordProperties.get(WorkflowAppWithScopedParameters.ONE_SPARK), oneSparkHistoryRuns.get(0).getPid());
    Assert.assertEquals(workflowRunRecordProperties.get(WorkflowAppWithScopedParameters.ANOTHER_MR), anotherMRHistoryRuns.get(0).getPid());
    Assert.assertEquals(workflowRunRecordProperties.get(WorkflowAppWithScopedParameters.ANOTHER_SPARK), anotherSparkHistoryRuns.get(0).getPid());
    // Get Workflow node states
    Map<String, WorkflowNodeStateDetail> nodeStates = getWorkflowNodeStates(programId, workflowHistoryRuns.get(0).getPid());
    Assert.assertNotNull(nodeStates);
    Assert.assertEquals(5, nodeStates.size());
    WorkflowNodeStateDetail mrNodeState = nodeStates.get(WorkflowAppWithScopedParameters.ONE_MR);
    Assert.assertNotNull(mrNodeState);
    Assert.assertEquals(WorkflowAppWithScopedParameters.ONE_MR, mrNodeState.getNodeId());
    Assert.assertEquals(oneMRHistoryRuns.get(0).getPid(), mrNodeState.getRunId());
    mrNodeState = nodeStates.get(WorkflowAppWithScopedParameters.ANOTHER_MR);
    Assert.assertNotNull(mrNodeState);
    Assert.assertEquals(WorkflowAppWithScopedParameters.ANOTHER_MR, mrNodeState.getNodeId());
    Assert.assertEquals(anotherMRHistoryRuns.get(0).getPid(), mrNodeState.getRunId());
    WorkflowNodeStateDetail sparkNodeState = nodeStates.get(WorkflowAppWithScopedParameters.ONE_SPARK);
    Assert.assertNotNull(sparkNodeState);
    Assert.assertEquals(WorkflowAppWithScopedParameters.ONE_SPARK, sparkNodeState.getNodeId());
    Assert.assertEquals(oneSparkHistoryRuns.get(0).getPid(), sparkNodeState.getRunId());
    sparkNodeState = nodeStates.get(WorkflowAppWithScopedParameters.ANOTHER_SPARK);
    Assert.assertNotNull(sparkNodeState);
    Assert.assertEquals(WorkflowAppWithScopedParameters.ANOTHER_SPARK, sparkNodeState.getNodeId());
    Assert.assertEquals(anotherSparkHistoryRuns.get(0).getPid(), sparkNodeState.getRunId());
    WorkflowNodeStateDetail oneActionNodeState = nodeStates.get(WorkflowAppWithScopedParameters.ONE_ACTION);
    Assert.assertNotNull(oneActionNodeState);
    Assert.assertEquals(WorkflowAppWithScopedParameters.ONE_ACTION, oneActionNodeState.getNodeId());
    // destroy method of the Workflow checks for field lineage operations and then add in the token
    // make sure the value in the token exists which means all checks in the destroy succeeded
    Id.Application applicationId = Id.Application.from(TEST_NAMESPACE2, WorkflowAppWithScopedParameters.APP_NAME);
    Id.Workflow workflowId = Id.Workflow.from(applicationId, WorkflowAppWithScopedParameters.ONE_WORKFLOW);
    WorkflowTokenDetail successToken = getWorkflowToken(workflowId, workflowHistoryRuns.get(0).getPid(), WorkflowToken.Scope.USER, WorkflowAppWithScopedParameters.SUCCESS_TOKEN_KEY);
    List<WorkflowTokenDetail.NodeValueDetail> nodeValue = successToken.getTokenData().values().iterator().next();
    Assert.assertEquals(nodeValue.get(0).getNode(), WorkflowAppWithScopedParameters.ONE_WORKFLOW);
    Assert.assertEquals(nodeValue.get(0).getValue(), WorkflowAppWithScopedParameters.SUCCESS_TOKEN_VALUE);
}