Examples with RunRecord co.cask.cdap.proto.RunRecord used on opensource projects

Example 31 with RunRecord

use of co.cask.cdap.proto.RunRecord in project cdap by caskdata.

the class WorkflowClientTestRun method testWorkflowClient.

@Test
public void testWorkflowClient() throws Exception {
    String keyValueTableType = "co.cask.cdap.api.dataset.lib.KeyValueTable";
    String filesetType = "co.cask.cdap.api.dataset.lib.FileSet";
    String outputPath = new File(TMP_FOLDER.newFolder(), "output").getAbsolutePath();
    Map<String, String> runtimeArgs = ImmutableMap.of("inputPath", createInput("input"), "outputPath", outputPath, "dataset.*.keep.local", "true");
    final WorkflowId workflowId = NamespaceId.DEFAULT.app(AppWithWorkflow.NAME).workflow(AppWithWorkflow.SampleWorkflow.NAME);
    programClient.start(workflowId, false, runtimeArgs);
    programClient.waitForStatus(workflowId, ProgramStatus.STOPPED, 60, TimeUnit.SECONDS);
    Tasks.waitFor(1, new Callable<Integer>() {

        @Override
        public Integer call() throws Exception {
            return programClient.getProgramRuns(workflowId, ProgramRunStatus.COMPLETED.name(), 0, Long.MAX_VALUE, 10).size();
        }
    }, 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    List<RunRecord> workflowRuns = programClient.getProgramRuns(workflowId, ProgramRunStatus.COMPLETED.name(), 0, Long.MAX_VALUE, 10);
    Assert.assertEquals(1, workflowRuns.size());
    String runId = workflowRuns.get(0).getPid();
    ProgramRunId workflowRunId = workflowId.run(runId);
    // Invalid test scenarios
    try {
        ProgramId nonExistentWorkflowId = new ProgramId(NamespaceId.DEFAULT.getNamespace(), AppWithWorkflow.NAME, ProgramType.WORKFLOW, "NonExistentWorkflow");
        ProgramRunId nonExistentWorkflowRun = nonExistentWorkflowId.run(runId);
        workflowClient.getWorkflowToken(nonExistentWorkflowRun);
        Assert.fail("Should not find a workflow token for a non-existing workflow");
    } catch (NotFoundException expected) {
    // expected
    }
    try {
        ProgramRunId invalidRunId = workflowId.run(RunIds.generate().getId());
        workflowClient.getWorkflowToken(invalidRunId);
        Assert.fail("Should not find a workflow token for a random run id");
    } catch (NotFoundException expected) {
    // expected
    }
    // Valid test scenarios
    WorkflowTokenDetail workflowToken = workflowClient.getWorkflowToken(workflowRunId);
    Assert.assertEquals(5, workflowToken.getTokenData().size());
    workflowToken = workflowClient.getWorkflowToken(workflowRunId, WorkflowToken.Scope.SYSTEM);
    Assert.assertTrue(workflowToken.getTokenData().size() > 0);
    workflowToken = workflowClient.getWorkflowToken(workflowRunId, "start_time");
    Map<String, List<WorkflowTokenDetail.NodeValueDetail>> tokenData = workflowToken.getTokenData();
    Assert.assertEquals(AppWithWorkflow.WordCountMapReduce.NAME, tokenData.get("start_time").get(0).getNode());
    Assert.assertTrue(Long.parseLong(tokenData.get("start_time").get(0).getValue()) < System.currentTimeMillis());
    workflowToken = workflowClient.getWorkflowToken(workflowRunId, WorkflowToken.Scope.USER, "action_type");
    tokenData = workflowToken.getTokenData();
    Assert.assertEquals(AppWithWorkflow.WordCountMapReduce.NAME, tokenData.get("action_type").get(0).getNode());
    Assert.assertEquals("MapReduce", tokenData.get("action_type").get(0).getValue());
    String nodeName = AppWithWorkflow.SampleWorkflow.FIRST_ACTION;
    WorkflowTokenNodeDetail workflowTokenAtNode = workflowClient.getWorkflowTokenAtNode(workflowRunId, nodeName);
    Assert.assertEquals(AppWithWorkflow.DummyAction.TOKEN_VALUE, workflowTokenAtNode.getTokenDataAtNode().get(AppWithWorkflow.DummyAction.TOKEN_KEY));
    workflowTokenAtNode = workflowClient.getWorkflowTokenAtNode(workflowRunId, nodeName, WorkflowToken.Scope.SYSTEM);
    Assert.assertEquals(0, workflowTokenAtNode.getTokenDataAtNode().size());
    workflowTokenAtNode = workflowClient.getWorkflowTokenAtNode(workflowRunId, nodeName, AppWithWorkflow.DummyAction.TOKEN_KEY);
    Assert.assertEquals(AppWithWorkflow.DummyAction.TOKEN_VALUE, workflowTokenAtNode.getTokenDataAtNode().get(AppWithWorkflow.DummyAction.TOKEN_KEY));
    String reduceOutputRecordsCounter = "org.apache.hadoop.mapreduce.TaskCounter.REDUCE_OUTPUT_RECORDS";
    workflowTokenAtNode = workflowClient.getWorkflowTokenAtNode(workflowRunId, AppWithWorkflow.WordCountMapReduce.NAME, WorkflowToken.Scope.SYSTEM, reduceOutputRecordsCounter);
    Assert.assertEquals(6, Integer.parseInt(workflowTokenAtNode.getTokenDataAtNode().get(reduceOutputRecordsCounter)));
    Map<String, DatasetSpecificationSummary> localDatasetSummaries = workflowClient.getWorkflowLocalDatasets(workflowRunId);
    Assert.assertEquals(2, localDatasetSummaries.size());
    DatasetSpecificationSummary keyValueTableSummary = new DatasetSpecificationSummary("MyTable." + runId, keyValueTableType, ImmutableMap.of("foo", "bar"));
    Assert.assertEquals(keyValueTableSummary, localDatasetSummaries.get("MyTable"));
    DatasetSpecificationSummary filesetSummary = new DatasetSpecificationSummary("MyFile." + runId, filesetType, ImmutableMap.of("anotherFoo", "anotherBar"));
    Assert.assertEquals(filesetSummary, localDatasetSummaries.get("MyFile"));
    workflowClient.deleteWorkflowLocalDatasets(workflowRunId);
    localDatasetSummaries = workflowClient.getWorkflowLocalDatasets(workflowRunId);
    Assert.assertEquals(0, localDatasetSummaries.size());
    Map<String, WorkflowNodeStateDetail> nodeStates = workflowClient.getWorkflowNodeStates(workflowRunId);
    Assert.assertEquals(3, nodeStates.size());
    WorkflowNodeStateDetail nodeState = nodeStates.get(AppWithWorkflow.SampleWorkflow.FIRST_ACTION);
    Assert.assertTrue(AppWithWorkflow.SampleWorkflow.FIRST_ACTION.equals(nodeState.getNodeId()));
    Assert.assertTrue(NodeStatus.COMPLETED == nodeState.getNodeStatus());
    nodeState = nodeStates.get(AppWithWorkflow.SampleWorkflow.SECOND_ACTION);
    Assert.assertTrue(AppWithWorkflow.SampleWorkflow.SECOND_ACTION.equals(nodeState.getNodeId()));
    Assert.assertTrue(NodeStatus.COMPLETED == nodeState.getNodeStatus());
    nodeState = nodeStates.get(AppWithWorkflow.SampleWorkflow.WORD_COUNT_MR);
    Assert.assertTrue(AppWithWorkflow.SampleWorkflow.WORD_COUNT_MR.equals(nodeState.getNodeId()));
    Assert.assertTrue(NodeStatus.COMPLETED == nodeState.getNodeStatus());
}

Example 32 with RunRecord

use of co.cask.cdap.proto.RunRecord in project cdap by caskdata.

the class ProgramClient method getProgramRuns.

/**
   * Gets the run records of a program.
   *
   * @param program the program
   * @param state - filter by status of the program
   * @return the run records of the program
   * @throws IOException if a network error occurred
   * @throws NotFoundException if the application or program could not be found
   * @throws UnauthenticatedException if the request is not authorized successfully in the gateway server
   */
public List<RunRecord> getProgramRuns(ProgramId program, String state, long startTime, long endTime, int limit) throws IOException, NotFoundException, UnauthenticatedException, UnauthorizedException {
    String queryParams = String.format("%s=%s&%s=%d&%s=%d&%s=%d", Constants.AppFabric.QUERY_PARAM_STATUS, state, Constants.AppFabric.QUERY_PARAM_START_TIME, startTime, Constants.AppFabric.QUERY_PARAM_END_TIME, endTime, Constants.AppFabric.QUERY_PARAM_LIMIT, limit);
    String path = String.format("apps/%s/versions/%s/%s/%s/runs?%s", program.getApplication(), program.getVersion(), program.getType().getCategoryName(), program.getProgram(), queryParams);
    URL url = config.resolveNamespacedURLV3(program.getNamespaceId(), path);
    HttpResponse response = restClient.execute(HttpMethod.GET, url, config.getAccessToken(), HttpURLConnection.HTTP_NOT_FOUND);
    if (response.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
        throw new NotFoundException(program);
    }
    return ObjectResponse.fromJsonBody(response, new TypeToken<List<RunRecord>>() {
    }).getResponseObject();
}

Example 33 with RunRecord

use of co.cask.cdap.proto.RunRecord in project cdap by caskdata.

the class WikipediaPipelineAppTest method getLatestPid.

@Nullable
private String getLatestPid(List<RunRecord> history) {
    String pid = null;
    long latestStartTime = 0;
    for (RunRecord runRecord : history) {
        // OK to use start ts, since we ensure that the next run begins after the previous run finishes in the test
        if (runRecord.getStartTs() > latestStartTime) {
            latestStartTime = runRecord.getStartTs();
            pid = runRecord.getPid();
        }
    }
    return pid;
}

Example 34 with RunRecord

use of co.cask.cdap.proto.RunRecord in project cdap by caskdata.

the class LogHandlerTestRun method testNextRunId.

private void testNextRunId(String appId, String entityType, String entityId, String namespace, String format, List<String> suppress) throws Exception {
    ProgramId programId = new NamespaceId(namespace).app(appId).program(ProgramType.valueOfCategoryName(entityType), entityId);
    RunRecord runRecord = mockLogReader.getRunRecord(programId);
    int expectedEvents = 20;
    if (runRecord.getStatus() == ProgramRunStatus.RUNNING || runRecord.getStatus() == ProgramRunStatus.SUSPENDED) {
        expectedEvents = 30;
    }
    String nextNoFromUrl;
    if (suppress.isEmpty()) {
        nextNoFromUrl = String.format("apps/%s/%s/%s/runs/%s/logs/next?format=%s&max=100", appId, entityType, entityId, runRecord.getPid(), format);
    } else {
        String fieldsToSuppress = getSuppressStr(suppress);
        nextNoFromUrl = String.format("apps/%s/%s/%s/runs/%s/logs/next?format=%s&max=100&suppress=%s", appId, entityType, entityId, runRecord.getPid(), format, fieldsToSuppress);
    }
    HttpResponse response = doGet(getVersionedAPIPath(nextNoFromUrl, namespace));
    verifyLogs(response, entityId, format, true, false, true, expectedEvents, 20, suppress);
}

Example 35 with RunRecord

use of co.cask.cdap.proto.RunRecord in project cdap by caskdata.

the class LogHandlerTestRun method testWorkflowRunLogs.

@Test
public void testWorkflowRunLogs() throws Exception {
    ProgramId workflowId = MockLogReader.SOME_WORKFLOW_APP.workflow(MockLogReader.SOME_WORKFLOW);
    RunRecord runRecord = mockLogReader.getRunRecord(workflowId);
    List<LogLine> logLines = getLogs(MockLogReader.TEST_NAMESPACE, MockLogReader.SOME_WORKFLOW_APP.getApplication(), "workflows", MockLogReader.SOME_WORKFLOW, runRecord.getPid(), "next");
    Assert.assertEquals(320, logLines.size());
    // First 80 lines correspond to the Workflow
    String log = logLines.get(5).getLog();
    Assert.assertTrue(log.contains(MockLogReader.SOME_WORKFLOW));
    Assert.assertFalse(log.contains(MockLogReader.SOME_MAPREDUCE));
    Assert.assertFalse(log.contains(MockLogReader.SOME_SPARK));
    // Lines 81-160 corresponds to MapReduce
    log = logLines.get(85).getLog();
    Assert.assertFalse(log.contains(MockLogReader.SOME_WORKFLOW));
    Assert.assertTrue(log.contains(MockLogReader.SOME_MAPREDUCE));
    Assert.assertFalse(log.contains(MockLogReader.SOME_SPARK));
    // Lines 161-240 corresponds to Spark
    log = logLines.get(165).getLog();
    Assert.assertFalse(log.contains(MockLogReader.SOME_WORKFLOW));
    Assert.assertFalse(log.contains(MockLogReader.SOME_MAPREDUCE));
    Assert.assertTrue(log.contains(MockLogReader.SOME_SPARK));
    ProgramId mapReduceId = MockLogReader.SOME_WORKFLOW_APP.mr(MockLogReader.SOME_MAPREDUCE);
    runRecord = mockLogReader.getRunRecord(mapReduceId);
    logLines = getLogs(MockLogReader.TEST_NAMESPACE, MockLogReader.SOME_WORKFLOW_APP.getApplication(), "mapreduce", MockLogReader.SOME_MAPREDUCE, runRecord.getPid(), "next");
    // Only 80 lines should correspond to MapReduce
    Assert.assertEquals(80, logLines.size());
    log = logLines.get(10).getLog();
    Assert.assertFalse(log.contains(MockLogReader.SOME_WORKFLOW));
    Assert.assertTrue(log.contains(MockLogReader.SOME_MAPREDUCE));
    Assert.assertFalse(log.contains(MockLogReader.SOME_SPARK));
    ProgramId sparkId = MockLogReader.SOME_WORKFLOW_APP.spark(MockLogReader.SOME_SPARK);
    runRecord = mockLogReader.getRunRecord(sparkId);
    logLines = getLogs(MockLogReader.TEST_NAMESPACE, MockLogReader.SOME_WORKFLOW_APP.getApplication(), "spark", MockLogReader.SOME_SPARK, runRecord.getPid(), "next");
    // Only 80 lines should correspond to Spark
    Assert.assertEquals(80, logLines.size());
    log = logLines.get(15).getLog();
    Assert.assertFalse(log.contains(MockLogReader.SOME_WORKFLOW));
    Assert.assertFalse(log.contains(MockLogReader.SOME_MAPREDUCE));
    Assert.assertTrue(log.contains(MockLogReader.SOME_SPARK));
}