Examples with WorkflowTokenDetail io.cdap.cdap.proto.WorkflowTokenDetail used on opensource projects

Example 1 with WorkflowTokenDetail

use of io.cdap.cdap.proto.WorkflowTokenDetail in project cdap by caskdata.

the class WorkflowClient method getWorkflowToken.

/**
 * Retrieve the {@link WorkflowToken} for the specified workflow run filtered by the specified
 * {@link WorkflowToken.Scope} and key.
 *
 * @param workflowRunId the run id of the workflow
 * @param scope the specified {@link WorkflowToken.Scope}. If null, it returns keys for
 * {@link WorkflowToken.Scope#USER}
 * @param key the specified key. If null, it returns all keys in the specified {@link WorkflowToken.Scope}
 * @return {@link WorkflowTokenDetail} with the specified filters
 */
public WorkflowTokenDetail getWorkflowToken(ProgramRunId workflowRunId, @Nullable WorkflowToken.Scope scope, @Nullable String key) throws IOException, UnauthenticatedException, NotFoundException, UnauthorizedException {
    String path = String.format("apps/%s/workflows/%s/runs/%s/token", workflowRunId.getApplication(), workflowRunId.getProgram(), workflowRunId.getRun());
    URL url = config.resolveNamespacedURLV3(workflowRunId.getNamespaceId(), appendScopeAndKeyToUrl(path, scope, key));
    HttpResponse response = restClient.execute(HttpMethod.GET, url, config.getAccessToken(), HttpURLConnection.HTTP_NOT_FOUND);
    if (response.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
        String msg = "Either the workflow or its run id";
        if (key != null) {
            msg = String.format("%s or the specified key at the specified scope", msg);
        }
        throw new NotFoundException(workflowRunId, msg);
    }
    return ObjectResponse.fromJsonBody(response, WorkflowTokenDetail.class, GSON).getResponseObject();
}

Example 2 with WorkflowTokenDetail

use of io.cdap.cdap.proto.WorkflowTokenDetail in project cdap by caskdata.

the class WorkflowHttpHandlerTest method testWorkflowToken.

@Test
@SuppressWarnings("ConstantConditions")
public void testWorkflowToken() throws Exception {
    deploy(AppWithWorkflow.class, 200);
    Id.Application appId = Id.Application.from(Id.Namespace.DEFAULT, AppWithWorkflow.NAME);
    final Id.Workflow workflowId = Id.Workflow.from(appId, AppWithWorkflow.SampleWorkflow.NAME);
    String outputPath = new File(tmpFolder.newFolder(), "output").getAbsolutePath();
    startProgram(workflowId, ImmutableMap.of("inputPath", createInput("input"), "outputPath", outputPath));
    Tasks.waitFor(1, () -> getProgramRuns(workflowId, ProgramRunStatus.COMPLETED).size(), 60, TimeUnit.SECONDS);
    List<RunRecord> programRuns = getProgramRuns(workflowId, ProgramRunStatus.COMPLETED);
    Assert.assertEquals(1, programRuns.size());
    RunRecord runRecord = programRuns.get(0);
    String pid = runRecord.getPid();
    // Verify entire worfklow token
    WorkflowTokenDetail workflowTokenDetail = getWorkflowToken(workflowId, pid, null, null);
    List<WorkflowTokenDetail.NodeValueDetail> nodeValueDetails = workflowTokenDetail.getTokenData().get(AppWithWorkflow.DummyAction.TOKEN_KEY);
    Assert.assertEquals(2, nodeValueDetails.size());
    Assert.assertEquals(AppWithWorkflow.SampleWorkflow.FIRST_ACTION, nodeValueDetails.get(0).getNode());
    Assert.assertEquals(AppWithWorkflow.SampleWorkflow.SECOND_ACTION, nodeValueDetails.get(1).getNode());
    Assert.assertEquals(AppWithWorkflow.DummyAction.TOKEN_VALUE, nodeValueDetails.get(0).getValue());
    Assert.assertEquals(AppWithWorkflow.DummyAction.TOKEN_VALUE, nodeValueDetails.get(1).getValue());
    // Verify entire workflow token by passing in the scope and key in the request
    workflowTokenDetail = getWorkflowToken(workflowId, pid, WorkflowToken.Scope.USER, AppWithWorkflow.DummyAction.TOKEN_KEY);
    nodeValueDetails = workflowTokenDetail.getTokenData().get(AppWithWorkflow.DummyAction.TOKEN_KEY);
    Assert.assertEquals(2, nodeValueDetails.size());
    Assert.assertEquals(AppWithWorkflow.SampleWorkflow.FIRST_ACTION, nodeValueDetails.get(0).getNode());
    Assert.assertEquals(AppWithWorkflow.SampleWorkflow.SECOND_ACTION, nodeValueDetails.get(1).getNode());
    Assert.assertEquals(AppWithWorkflow.DummyAction.TOKEN_VALUE, nodeValueDetails.get(0).getValue());
    Assert.assertEquals(AppWithWorkflow.DummyAction.TOKEN_VALUE, nodeValueDetails.get(1).getValue());
    // Get workflow level tokens
    WorkflowTokenNodeDetail nodeDetail = getWorkflowToken(workflowId, pid, AppWithWorkflow.SampleWorkflow.NAME, WorkflowToken.Scope.USER, null);
    Map<String, String> tokenData = nodeDetail.getTokenDataAtNode();
    Assert.assertEquals(2, tokenData.size());
    Assert.assertEquals(AppWithWorkflow.SampleWorkflow.INITIALIZE_TOKEN_VALUE, tokenData.get(AppWithWorkflow.SampleWorkflow.INITIALIZE_TOKEN_KEY));
    Assert.assertEquals(AppWithWorkflow.SampleWorkflow.DESTROY_TOKEN_SUCCESS_VALUE, tokenData.get(AppWithWorkflow.SampleWorkflow.DESTROY_TOKEN_KEY));
    // Verify workflow token at a given node
    WorkflowTokenNodeDetail tokenAtNode = getWorkflowToken(workflowId, pid, AppWithWorkflow.SampleWorkflow.FIRST_ACTION, null, null);
    Map<String, String> tokenDataAtNode = tokenAtNode.getTokenDataAtNode();
    Assert.assertEquals(1, tokenDataAtNode.size());
    Assert.assertEquals(AppWithWorkflow.DummyAction.TOKEN_VALUE, tokenDataAtNode.get(AppWithWorkflow.DummyAction.TOKEN_KEY));
    // Verify workflow token at a given node by passing in a scope and a key
    tokenAtNode = getWorkflowToken(workflowId, pid, AppWithWorkflow.SampleWorkflow.FIRST_ACTION, WorkflowToken.Scope.USER, AppWithWorkflow.DummyAction.TOKEN_KEY);
    tokenDataAtNode = tokenAtNode.getTokenDataAtNode();
    Assert.assertEquals(1, tokenDataAtNode.size());
    Assert.assertEquals(AppWithWorkflow.DummyAction.TOKEN_VALUE, tokenDataAtNode.get(AppWithWorkflow.DummyAction.TOKEN_KEY));
}

Example 3 with WorkflowTokenDetail

use of io.cdap.cdap.proto.WorkflowTokenDetail in project cdap by caskdata.

the class AppFabricClient method getWorkflowToken.

public WorkflowTokenDetail getWorkflowToken(String namespaceId, String appId, String wflowId, String runId, @Nullable WorkflowToken.Scope scope, @Nullable String key) throws NotFoundException, UnauthorizedException {
    MockResponder responder = new MockResponder();
    String uri = String.format("%s/apps/%s/workflows/%s/runs/%s/token", getNamespacePath(namespaceId), appId, wflowId, runId);
    HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, uri);
    scope = scope == null ? WorkflowToken.Scope.USER : scope;
    key = key == null ? "" : key;
    workflowHttpHandler.getWorkflowToken(request, responder, namespaceId, appId, wflowId, runId, scope.name(), key);
    Type workflowTokenDetailType = new TypeToken<WorkflowTokenDetail>() {
    }.getType();
    WorkflowTokenDetail workflowTokenDetail = responder.decodeResponseContent(workflowTokenDetailType, GSON);
    verifyResponse(HttpResponseStatus.OK, responder.getStatus(), "Getting workflow token failed");
    return workflowTokenDetail;
}

Example 4 with WorkflowTokenDetail

use of io.cdap.cdap.proto.WorkflowTokenDetail in project cdap by caskdata.

the class TestFrameworkTestRun method testDeployWorkflowApp.

@Category(XSlowTests.class)
@Test
@Ignore
public void testDeployWorkflowApp() throws Exception {
    // Add test back when CDAP-12350 is resolved
    ApplicationManager applicationManager = deployApplication(testSpace, AppWithSchedule.class);
    final WorkflowManager wfmanager = applicationManager.getWorkflowManager(AppWithSchedule.WORKFLOW_NAME);
    List<ScheduleDetail> schedules = wfmanager.getProgramSchedules();
    Assert.assertEquals(2, schedules.size());
    String scheduleName = schedules.get(1).getName();
    Assert.assertNotNull(scheduleName);
    Assert.assertFalse(scheduleName.isEmpty());
    final int initialRuns = wfmanager.getHistory().size();
    LOG.info("initialRuns = {}", initialRuns);
    wfmanager.getSchedule(scheduleName).resume();
    String status = wfmanager.getSchedule(scheduleName).status(200);
    Assert.assertEquals("SCHEDULED", status);
    // Make sure something ran before suspending
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return wfmanager.getHistory().size() > 0;
        }
    }, 15, TimeUnit.SECONDS);
    wfmanager.getSchedule(scheduleName).suspend();
    waitForScheduleState(scheduleName, wfmanager, ProgramScheduleStatus.SUSPENDED);
    // All runs should be completed
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            for (RunRecord record : wfmanager.getHistory()) {
                if (record.getStatus() != ProgramRunStatus.COMPLETED) {
                    return false;
                }
            }
            return true;
        }
    }, 15, TimeUnit.SECONDS);
    List<RunRecord> history = wfmanager.getHistory();
    int workflowRuns = history.size();
    LOG.info("workflowRuns = {}", workflowRuns);
    Assert.assertTrue(workflowRuns > 0);
    // Sleep for some time and verify there are no more scheduled jobs after the suspend.
    TimeUnit.SECONDS.sleep(5);
    final int workflowRunsAfterSuspend = wfmanager.getHistory().size();
    Assert.assertEquals(workflowRuns, workflowRunsAfterSuspend);
    wfmanager.getSchedule(scheduleName).resume();
    // Check that after resume it goes to "SCHEDULED" state
    waitForScheduleState(scheduleName, wfmanager, ProgramScheduleStatus.SCHEDULED);
    // Make sure new runs happens after resume
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return wfmanager.getHistory().size() > workflowRunsAfterSuspend;
        }
    }, 15, TimeUnit.SECONDS);
    // Check scheduled state
    Assert.assertEquals("SCHEDULED", wfmanager.getSchedule(scheduleName).status(200));
    // Check status of non-existent schedule
    Assert.assertEquals("NOT_FOUND", wfmanager.getSchedule("doesnt exist").status(404));
    // Suspend the schedule
    wfmanager.getSchedule(scheduleName).suspend();
    // Check that after suspend it goes to "SUSPENDED" state
    waitForScheduleState(scheduleName, wfmanager, ProgramScheduleStatus.SUSPENDED);
    // Test workflow token while suspended
    String pid = history.get(0).getPid();
    WorkflowTokenDetail workflowToken = wfmanager.getToken(pid, WorkflowToken.Scope.SYSTEM, null);
    Assert.assertEquals(0, workflowToken.getTokenData().size());
    workflowToken = wfmanager.getToken(pid, null, null);
    Assert.assertEquals(2, workflowToken.getTokenData().size());
    // Wait for all workflow runs to finish execution, in case more than one run happened with an enabled schedule
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            for (RunRecord record : wfmanager.getHistory()) {
                if (record.getStatus() != ProgramRunStatus.COMPLETED) {
                    return false;
                }
            }
            return true;
        }
    }, 15, TimeUnit.SECONDS);
    // Verify workflow token after workflow completion
    WorkflowTokenNodeDetail workflowTokenAtNode = wfmanager.getTokenAtNode(pid, AppWithSchedule.DummyAction.class.getSimpleName(), WorkflowToken.Scope.USER, "finished");
    Assert.assertEquals(true, Boolean.parseBoolean(workflowTokenAtNode.getTokenDataAtNode().get("finished")));
    workflowToken = wfmanager.getToken(pid, null, null);
    Assert.assertEquals(false, Boolean.parseBoolean(workflowToken.getTokenData().get("running").get(0).getValue()));
}

Example 5 with WorkflowTokenDetail

use of io.cdap.cdap.proto.WorkflowTokenDetail in project cdap by caskdata.

the class WorkflowHttpHandlerTest method testWorkflowTokenPut.

@Test
public void testWorkflowTokenPut() throws Exception {
    deploy(WorkflowTokenTestPutApp.class, 200);
    Id.Application appId = Id.Application.from(Id.Namespace.DEFAULT, WorkflowTokenTestPutApp.NAME);
    Id.Workflow workflowId = Id.Workflow.from(appId, WorkflowTokenTestPutApp.WorkflowTokenTestPut.NAME);
    Id.Program sparkId = Id.Program.from(appId, ProgramType.SPARK, WorkflowTokenTestPutApp.SparkTestApp.NAME);
    // Start program with inputPath and outputPath arguments.
    // This should succeed. The programs inside the workflow will attempt to write to the workflow token
    // from the Mapper's and Reducer's methods as well as from a Spark closure, and they will throw an exception
    // if that succeeds.
    // The MapReduce's initialize will record the workflow run id in the token, and the destroy as well
    // as the mapper and the reducer will validate that they have the same workflow run id.
    String outputPath = new File(tmpFolder.newFolder(), "output").getAbsolutePath();
    startProgram(workflowId, ImmutableMap.of("inputPath", createInputForRecordVerification("sixthInput"), "outputPath", outputPath));
    waitState(workflowId, ProgramStatus.RUNNING.name());
    waitState(workflowId, ProgramStatus.STOPPED.name());
    // validate the completed workflow run and validate that it is the same as recorded in the token
    verifyProgramRuns(workflowId, ProgramRunStatus.COMPLETED);
    List<RunRecord> runs = getProgramRuns(workflowId, ProgramRunStatus.COMPLETED);
    Assert.assertEquals(1, runs.size());
    String wfRunId = runs.get(0).getPid();
    WorkflowTokenDetail tokenDetail = getWorkflowToken(workflowId, wfRunId, null, null);
    List<WorkflowTokenDetail.NodeValueDetail> details = tokenDetail.getTokenData().get("wf.runid");
    Assert.assertEquals(1, details.size());
    Assert.assertEquals(wfRunId, details.get(0).getValue());
    // validate that none of the mapper, reducer or spark closure were able to write to the token
    for (String key : new String[] { "mapper.initialize.key", "map.key", "reducer.initialize.key", "reduce.key", "some.key" }) {
        Assert.assertFalse(tokenDetail.getTokenData().containsKey(key));
    }
    List<RunRecord> sparkProgramRuns = getProgramRuns(sparkId, ProgramRunStatus.COMPLETED);
    Assert.assertEquals(1, sparkProgramRuns.size());
}