Examples with WorkflowId io.cdap.cdap.proto.id.WorkflowId used on opensource projects

Example 11 with WorkflowId

use of io.cdap.cdap.proto.id.WorkflowId in project cdap by caskdata.

the class WorkflowHttpHandlerTest method testKillSuspendedWorkflow.

@Category(XSlowTests.class)
@Test
public void testKillSuspendedWorkflow() throws Exception {
    deploy(SleepingWorkflowApp.class, 200, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);
    WorkflowId workflow = new WorkflowId(TEST_NAMESPACE2, "SleepWorkflowApp", "SleepWorkflow");
    // Start the workflow, with max long as sleep time. This make the workflow never complete by itself.
    startProgram(workflow, Collections.singletonMap("sleep.ms", Long.toString(Long.MAX_VALUE)), 200);
    waitState(workflow, ProgramStatus.RUNNING.name());
    String runId = getRunIdOfRunningProgram(Id.Program.fromEntityId(workflow));
    suspendWorkflow(Id.Program.fromEntityId(workflow), runId, 200);
    // Workflow status should be SUSPENDED
    verifyProgramRuns(workflow, ProgramRunStatus.SUSPENDED);
    stopProgram(Id.Program.fromEntityId(workflow), runId, 200);
    waitState(workflow, ProgramStatus.STOPPED.name());
    verifyProgramRuns(Id.Program.fromEntityId(workflow), ProgramRunStatus.KILLED, 0);
}

Example 12 with WorkflowId

use of io.cdap.cdap.proto.id.WorkflowId in project cdap by caskdata.

the class UpgradeJobMain method suspendSchedulesAndStopPipelines.

private static void suspendSchedulesAndStopPipelines(ClientConfig clientConfig) throws Exception {
    ApplicationClient applicationClient = new ApplicationClient(clientConfig);
    ScheduleClient scheduleClient = new ScheduleClient(clientConfig);
    ProgramClient programClient = new ProgramClient(clientConfig);
    NamespaceClient namespaceClient = new NamespaceClient(clientConfig);
    boolean shouldRetry = false;
    List<NamespaceId> namespaceIdList = namespaceClient.list().stream().map(NamespaceMeta::getNamespaceId).collect(Collectors.toList());
    namespaceIdList.add(NamespaceId.SYSTEM);
    for (NamespaceId namespaceId : namespaceIdList) {
        for (ApplicationRecord record : applicationClient.list(namespaceId)) {
            ApplicationId applicationId = new ApplicationId(namespaceId.getNamespace(), record.getName(), record.getAppVersion());
            LOG.debug("Trying to stop schedule and workflows for application " + applicationId);
            List<WorkflowId> workflowIds = applicationClient.get(applicationId).getPrograms().stream().filter(programRecord -> programRecord.getType().equals(ProgramType.WORKFLOW)).map(programRecord -> new WorkflowId(applicationId, programRecord.getName())).collect(Collectors.toList());
            for (WorkflowId workflowId : workflowIds) {
                List<ScheduleId> scheduleIds = scheduleClient.listSchedules(workflowId).stream().map(scheduleDetail -> new ScheduleId(namespaceId.getNamespace(), record.getName(), record.getAppVersion(), scheduleDetail.getName())).collect(Collectors.toList());
                for (ScheduleId scheduleId : scheduleIds) {
                    if (scheduleClient.getStatus(scheduleId).equals(SCHEDULED)) {
                        scheduleClient.suspend(scheduleId);
                    }
                }
                // Need to stop workflows first or else the program will fail to stop below
                if (!programClient.getStatus(workflowId).equals(ProgramStatus.STOPPED.toString())) {
                    try {
                        programClient.stop(workflowId);
                    } catch (BadRequestException e) {
                        // transitioned to stop state since it was checked earlier or not.
                        if (!programClient.getStatus(workflowId).equals(ProgramStatus.STOPPED.toString())) {
                            // Pipeline still in running state. Continue with stopping rest of the pipelines in this namespace and
                            // next retry should try to stop/verify status for this pipeline.
                            shouldRetry = true;
                        }
                    }
                }
            }
        }
        // At least one pipeline is still in running state so retry to verify pipeline status .
        if (shouldRetry) {
            throw new RetryableException("At least one pipeline in namespace " + namespaceId + " is still running.");
        }
        // All schedules are stopped, now stop all programs
        programClient.stopAll(namespaceId);
    }
}

Example 13 with WorkflowId

use of io.cdap.cdap.proto.id.WorkflowId in project cdap by caskdata.

the class WorkflowClientTestRun method testWorkflowClient.

@Test
public void testWorkflowClient() throws Exception {
    String keyValueTableType = "io.cdap.cdap.api.dataset.lib.KeyValueTable";
    String filesetType = "io.cdap.cdap.api.dataset.lib.FileSet";
    String outputPath = new File(TMP_FOLDER.newFolder(), "output").getAbsolutePath();
    Map<String, String> runtimeArgs = ImmutableMap.of("inputPath", createInput("input"), "outputPath", outputPath, "dataset.*.keep.local", "true");
    final WorkflowId workflowId = NamespaceId.DEFAULT.app(AppWithWorkflow.NAME).workflow(AppWithWorkflow.SampleWorkflow.NAME);
    programClient.start(workflowId, false, runtimeArgs);
    assertProgramRuns(programClient, workflowId, ProgramRunStatus.COMPLETED, 1, 120);
    List<RunRecord> workflowRuns = programClient.getProgramRuns(workflowId, ProgramRunStatus.COMPLETED.name(), 0, Long.MAX_VALUE, 10);
    Assert.assertEquals(1, workflowRuns.size());
    String runId = workflowRuns.get(0).getPid();
    ProgramRunId workflowRunId = workflowId.run(runId);
    // Invalid test scenarios
    try {
        ProgramId nonExistentWorkflowId = new ProgramId(NamespaceId.DEFAULT.getNamespace(), AppWithWorkflow.NAME, ProgramType.WORKFLOW, "NonExistentWorkflow");
        ProgramRunId nonExistentWorkflowRun = nonExistentWorkflowId.run(runId);
        workflowClient.getWorkflowToken(nonExistentWorkflowRun);
        Assert.fail("Should not find a workflow token for a non-existing workflow");
    } catch (NotFoundException expected) {
    // expected
    }
    try {
        ProgramRunId invalidRunId = workflowId.run(RunIds.generate().getId());
        workflowClient.getWorkflowToken(invalidRunId);
        Assert.fail("Should not find a workflow token for a random run id");
    } catch (NotFoundException expected) {
    // expected
    }
    // Valid test scenarios
    WorkflowTokenDetail workflowToken = workflowClient.getWorkflowToken(workflowRunId);
    Assert.assertEquals(5, workflowToken.getTokenData().size());
    workflowToken = workflowClient.getWorkflowToken(workflowRunId, WorkflowToken.Scope.SYSTEM);
    Assert.assertTrue(workflowToken.getTokenData().size() > 0);
    workflowToken = workflowClient.getWorkflowToken(workflowRunId, "start_time");
    Map<String, List<WorkflowTokenDetail.NodeValueDetail>> tokenData = workflowToken.getTokenData();
    Assert.assertEquals(AppWithWorkflow.WordCountMapReduce.NAME, tokenData.get("start_time").get(0).getNode());
    Assert.assertTrue(Long.parseLong(tokenData.get("start_time").get(0).getValue()) < System.currentTimeMillis());
    workflowToken = workflowClient.getWorkflowToken(workflowRunId, WorkflowToken.Scope.USER, "action_type");
    tokenData = workflowToken.getTokenData();
    Assert.assertEquals(AppWithWorkflow.WordCountMapReduce.NAME, tokenData.get("action_type").get(0).getNode());
    Assert.assertEquals("MapReduce", tokenData.get("action_type").get(0).getValue());
    String nodeName = AppWithWorkflow.SampleWorkflow.FIRST_ACTION;
    WorkflowTokenNodeDetail workflowTokenAtNode = workflowClient.getWorkflowTokenAtNode(workflowRunId, nodeName);
    Assert.assertEquals(AppWithWorkflow.DummyAction.TOKEN_VALUE, workflowTokenAtNode.getTokenDataAtNode().get(AppWithWorkflow.DummyAction.TOKEN_KEY));
    workflowTokenAtNode = workflowClient.getWorkflowTokenAtNode(workflowRunId, nodeName, WorkflowToken.Scope.SYSTEM);
    Assert.assertEquals(0, workflowTokenAtNode.getTokenDataAtNode().size());
    workflowTokenAtNode = workflowClient.getWorkflowTokenAtNode(workflowRunId, nodeName, AppWithWorkflow.DummyAction.TOKEN_KEY);
    Assert.assertEquals(AppWithWorkflow.DummyAction.TOKEN_VALUE, workflowTokenAtNode.getTokenDataAtNode().get(AppWithWorkflow.DummyAction.TOKEN_KEY));
    String reduceOutputRecordsCounter = "org.apache.hadoop.mapreduce.TaskCounter.REDUCE_OUTPUT_RECORDS";
    workflowTokenAtNode = workflowClient.getWorkflowTokenAtNode(workflowRunId, AppWithWorkflow.WordCountMapReduce.NAME, WorkflowToken.Scope.SYSTEM, reduceOutputRecordsCounter);
    Assert.assertEquals(6, Integer.parseInt(workflowTokenAtNode.getTokenDataAtNode().get(reduceOutputRecordsCounter)));
    Map<String, DatasetSpecificationSummary> localDatasetSummaries = workflowClient.getWorkflowLocalDatasets(workflowRunId);
    Assert.assertEquals(2, localDatasetSummaries.size());
    DatasetSpecificationSummary keyValueTableSummary = new DatasetSpecificationSummary("MyTable." + runId, keyValueTableType, ImmutableMap.of("foo", "bar"));
    Assert.assertEquals(keyValueTableSummary, localDatasetSummaries.get("MyTable"));
    DatasetSpecificationSummary filesetSummary = new DatasetSpecificationSummary("MyFile." + runId, filesetType, ImmutableMap.of("anotherFoo", "anotherBar"));
    Assert.assertEquals(filesetSummary, localDatasetSummaries.get("MyFile"));
    workflowClient.deleteWorkflowLocalDatasets(workflowRunId);
    localDatasetSummaries = workflowClient.getWorkflowLocalDatasets(workflowRunId);
    Assert.assertEquals(0, localDatasetSummaries.size());
    Map<String, WorkflowNodeStateDetail> nodeStates = workflowClient.getWorkflowNodeStates(workflowRunId);
    Assert.assertEquals(3, nodeStates.size());
    WorkflowNodeStateDetail nodeState = nodeStates.get(AppWithWorkflow.SampleWorkflow.FIRST_ACTION);
    Assert.assertTrue(AppWithWorkflow.SampleWorkflow.FIRST_ACTION.equals(nodeState.getNodeId()));
    Assert.assertTrue(NodeStatus.COMPLETED == nodeState.getNodeStatus());
    nodeState = nodeStates.get(AppWithWorkflow.SampleWorkflow.SECOND_ACTION);
    Assert.assertTrue(AppWithWorkflow.SampleWorkflow.SECOND_ACTION.equals(nodeState.getNodeId()));
    Assert.assertTrue(NodeStatus.COMPLETED == nodeState.getNodeStatus());
    nodeState = nodeStates.get(AppWithWorkflow.SampleWorkflow.WORD_COUNT_MR);
    Assert.assertTrue(AppWithWorkflow.SampleWorkflow.WORD_COUNT_MR.equals(nodeState.getNodeId()));
    Assert.assertTrue(NodeStatus.COMPLETED == nodeState.getNodeStatus());
}

Example 14 with WorkflowId

use of io.cdap.cdap.proto.id.WorkflowId in project cdap by caskdata.

the class WorkflowStatsSLAHttpHandler method workflowRunDetail.

/**
 * The endpoint returns a list of workflow metrics based on the workflow run and a surrounding number of runs
 * of the workflow that are spaced apart by a time interval from each other.
 *
 * @param request The request
 * @param responder The responder
 * @param namespaceId The namespace the application is in
 * @param appId The application the workflow is in
 * @param workflowId The workflow that needs to have it stats shown
 * @param runId The run id of the Workflow that the user wants to see
 * @param limit The number of the records that the user wants to compare against on either side of the run
 * @param interval The timeInterval with which the user wants to space out the runs
 */
@GET
@Path("apps/{app-id}/workflows/{workflow-id}/runs/{run-id}/statistics")
public void workflowRunDetail(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("app-id") String appId, @PathParam("workflow-id") String workflowId, @PathParam("run-id") String runId, @QueryParam("limit") @DefaultValue("10") int limit, @QueryParam("interval") @DefaultValue("10s") String interval) throws Exception {
    if (limit <= 0) {
        throw new BadRequestException("Limit has to be greater than 0. Entered value was : " + limit);
    }
    long timeInterval;
    try {
        timeInterval = TimeMathParser.resolutionInSeconds(interval);
    } catch (IllegalArgumentException e) {
        throw new BadRequestException("Interval is specified with invalid time unit. It should be specified with one" + " of the 'ms', 's', 'm', 'h', 'd' units. Entered value was : " + interval);
    }
    if (timeInterval <= 0) {
        throw new BadRequestException("Interval should be greater than 0 and should be specified with one of the 'ms'," + " 's', 'm', 'h', 'd' units. Entered value was : " + interval);
    }
    WorkflowId workflow = new WorkflowId(namespaceId, appId, workflowId);
    Collection<WorkflowTable.WorkflowRunRecord> workflowRunRecords = store.retrieveSpacedRecords(workflow, runId, limit, timeInterval);
    List<WorkflowRunMetrics> workflowRunMetricsList = new ArrayList<>();
    Map<String, Long> startTimes = new HashMap<>();
    for (WorkflowTable.WorkflowRunRecord workflowRunRecord : workflowRunRecords) {
        workflowRunMetricsList.add(getDetailedRecord(workflow, workflowRunRecord.getWorkflowRunId()));
        startTimes.put(workflowRunRecord.getWorkflowRunId(), RunIds.getTime(RunIds.fromString(workflowRunRecord.getWorkflowRunId()), TimeUnit.SECONDS));
    }
    Collection<WorkflowStatsComparison.ProgramNodes> formattedStatisticsMap = format(workflowRunMetricsList);
    responder.sendJson(HttpResponseStatus.OK, GSON.toJson(new WorkflowStatsComparison(startTimes, formattedStatisticsMap)));
}

Example 15 with WorkflowId

use of io.cdap.cdap.proto.id.WorkflowId in project cdap by caskdata.

the class WorkflowStatsSLAHttpHandler method workflowStats.

/**
 * Returns the statistics for a given workflow.
 *
 * @param request The request
 * @param responder The responder
 * @param namespaceId The namespace the application is in
 * @param appId The application the workflow is in
 * @param workflowId The workflow that needs to have it stats shown
 * @param start The start time of the range
 * @param end The end time of the range
 * @param percentiles The list of percentile values on which visibility is needed
 */
@GET
@Path("apps/{app-id}/workflows/{workflow-id}/statistics")
public void workflowStats(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @PathParam("app-id") String appId, @PathParam("workflow-id") String workflowId, @QueryParam("start") @DefaultValue("now-1d") String start, @QueryParam("end") @DefaultValue("now") String end, @QueryParam("percentile") @DefaultValue("90.0") List<Double> percentiles) throws Exception {
    long startTime = TimeMathParser.parseTimeInSeconds(start);
    long endTime = TimeMathParser.parseTimeInSeconds(end);
    if (startTime < 0) {
        throw new BadRequestException("Invalid start time. The time you entered was : " + startTime);
    } else if (endTime < 0) {
        throw new BadRequestException("Invalid end time. The time you entered was : " + endTime);
    } else if (endTime < startTime) {
        throw new BadRequestException("Start time : " + startTime + " cannot be larger than end time : " + endTime);
    }
    for (double i : percentiles) {
        if (i < 0.0 || i > 100.0) {
            throw new BadRequestException("Percentile values have to be greater than or equal to 0 and" + " less than or equal to 100. Invalid input was " + Double.toString(i));
        }
    }
    WorkflowId workflow = new WorkflowId(namespaceId, appId, workflowId);
    WorkflowStatistics workflowStatistics = store.getWorkflowStatistics(workflow, startTime, endTime, percentiles);
    if (workflowStatistics == null) {
        responder.sendString(HttpResponseStatus.OK, "There are no statistics associated with this workflow : " + workflowId + " in the specified time range.");
        return;
    }
    responder.sendJson(HttpResponseStatus.OK, GSON.toJson(workflowStatistics));
}