use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.
the class WorkflowHttpHandlerTest method testWorkflowForkApp.
@Test
public void testWorkflowForkApp() throws Exception {
File directory = tmpFolder.newFolder();
Map<String, String> runtimeArgs = new HashMap<>();
// Files used to synchronize between this test and workflow execution
File firstFile = new File(directory, "first.file");
File firstDoneFile = new File(directory, "first.done");
runtimeArgs.put("first.file", firstFile.getAbsolutePath());
runtimeArgs.put("first.donefile", firstDoneFile.getAbsolutePath());
File branch1File = new File(directory, "branch1.file");
File branch1DoneFile = new File(directory, "branch1.done");
runtimeArgs.put("branch1.file", branch1File.getAbsolutePath());
runtimeArgs.put("branch1.donefile", branch1DoneFile.getAbsolutePath());
File branch2File = new File(directory, "branch2.file");
File branch2DoneFile = new File(directory, "branch2.done");
runtimeArgs.put("branch2.file", branch2File.getAbsolutePath());
runtimeArgs.put("branch2.donefile", branch2DoneFile.getAbsolutePath());
HttpResponse response = deploy(WorkflowAppWithFork.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
Id.Program programId = Id.Program.from(TEST_NAMESPACE2, WorkflowAppWithFork.class.getSimpleName(), ProgramType.WORKFLOW, WorkflowAppWithFork.WorkflowWithFork.class.getSimpleName());
setAndTestRuntimeArgs(programId, runtimeArgs);
// Start a Workflow
startProgram(programId);
// Workflow should be running
waitState(programId, ProgramStatus.RUNNING.name());
// Get the runId for the currently running Workflow
String runId = getRunIdOfRunningProgram(programId);
// Wait until first action in the Workflow starts executing
verifyFileExists(Lists.newArrayList(firstFile));
verifyRunningProgramCount(programId, runId, 1);
// Stop the Workflow
stopProgram(programId);
// Workflow run record should be marked 'killed'
verifyProgramRuns(programId, "killed");
// Delete the asset created in the previous run
Assert.assertTrue(firstFile.delete());
// Start the Workflow again
startProgram(programId);
// Workflow should be running
waitState(programId, ProgramStatus.RUNNING.name());
// Get the runId for the currently running Workflow
String newRunId = getRunIdOfRunningProgram(programId);
Assert.assertTrue(String.format("Expected a new runId to be generated after starting the workflow for the second time, but " + "found old runId '%s' = new runId '%s'", runId, newRunId), !runId.equals(newRunId));
// Store the new RunId
runId = newRunId;
// Wait until first action in the Workflow starts executing
verifyFileExists(Lists.newArrayList(firstFile));
verifyRunningProgramCount(programId, runId, 1);
// Signal the first action to continue
Assert.assertTrue(firstDoneFile.createNewFile());
// Wait until fork in the Workflow starts executing
verifyFileExists(Lists.newArrayList(branch1File, branch2File));
// Two actions should be running in Workflow as a part of the fork
verifyRunningProgramCount(programId, runId, 2);
// Stop the program while in fork
stopProgram(programId, 200);
// Wait until the program stop
waitState(programId, ProgramStatus.STOPPED.name());
// Current endpoint would return 404
response = getWorkflowCurrentStatus(programId, runId);
Assert.assertEquals(404, response.getStatusLine().getStatusCode());
// Now there should be 2 RunRecord with status killed
verifyProgramRuns(programId, "killed", 1);
// Delete the assets generated in the previous run
Assert.assertTrue(firstFile.delete());
Assert.assertTrue(firstDoneFile.delete());
Assert.assertTrue(branch1File.delete());
Assert.assertTrue(branch2File.delete());
// Restart the run again
startProgram(programId);
// Wait until the Workflow is running
waitState(programId, ProgramStatus.RUNNING.name());
// Store the new RunRecord for the currently running run
runId = getRunIdOfRunningProgram(programId);
// Wait until first action in the Workflow starts executing
verifyFileExists(Lists.newArrayList(firstFile));
verifyRunningProgramCount(programId, runId, 1);
// Signal the first action to continue
Assert.assertTrue(firstDoneFile.createNewFile());
// Wait until fork in the Workflow starts executing
verifyFileExists(Lists.newArrayList(branch1File, branch2File));
// Two actions should be running in Workflow as a part of the fork
verifyRunningProgramCount(programId, runId, 2);
// Signal the Workflow that execution can be continued
Assert.assertTrue(branch1DoneFile.createNewFile());
Assert.assertTrue(branch2DoneFile.createNewFile());
// Workflow should now have one completed run
verifyProgramRuns(programId, "completed");
}
use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.
the class WorkflowHttpHandlerTest method testWorkflowRuns.
@Test
public void testWorkflowRuns() throws Exception {
String appName = "WorkflowAppWithErrorRuns";
String workflowName = "WorkflowWithErrorRuns";
HttpResponse response = deploy(WorkflowAppWithErrorRuns.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
Id.Program programId = Id.Program.from(TEST_NAMESPACE2, appName, ProgramType.WORKFLOW, workflowName);
// Test the "KILLED" state of the Workflow.
File instance1File = new File(tmpFolder.newFolder() + "/instance1.file");
File instance2File = new File(tmpFolder.newFolder() + "/instance2.file");
File doneFile = new File(tmpFolder.newFolder() + "/done.file");
// Start the first Workflow run.
Map<String, String> propertyMap = ImmutableMap.of("simple.action.file", instance1File.getAbsolutePath(), "simple.action.donefile", doneFile.getAbsolutePath());
startProgram(programId, propertyMap);
// Start another Workflow run.
propertyMap = ImmutableMap.of("simple.action.file", instance2File.getAbsolutePath(), "simple.action.donefile", doneFile.getAbsolutePath());
startProgram(programId, propertyMap);
// Wait until the execution of actions in both Workflow runs is started.
while (!(instance1File.exists() && instance2File.exists())) {
TimeUnit.MILLISECONDS.sleep(50);
}
// Verify that there are two runs of the Workflow currently running.
List<RunRecord> historyRuns = getProgramRuns(programId, "running");
Assert.assertEquals(2, historyRuns.size());
// Stop both Workflow runs.
String runId = historyRuns.get(0).getPid();
stopProgram(programId, runId, 200);
runId = historyRuns.get(1).getPid();
stopProgram(programId, runId, 200);
// Verify both runs should be marked "KILLED".
verifyProgramRuns(programId, "killed", 1);
// Test the "COMPLETE" state of the Workflow.
File instanceFile = new File(tmpFolder.newFolder() + "/instance.file");
propertyMap = ImmutableMap.of("simple.action.file", instanceFile.getAbsolutePath(), "simple.action.donefile", doneFile.getAbsolutePath());
startProgram(programId, propertyMap);
while (!instanceFile.exists()) {
TimeUnit.MILLISECONDS.sleep(50);
}
// Verify that currently only one run of the Workflow should be running.
historyRuns = getProgramRuns(programId, "running");
Assert.assertEquals(1, historyRuns.size());
Assert.assertTrue(doneFile.createNewFile());
// Verify that Workflow should move to "COMPLETED" state.
verifyProgramRuns(programId, "completed");
// Test the "FAILED" state of the program.
propertyMap = ImmutableMap.of("ThrowError", "true");
startProgram(programId, propertyMap);
// Verify that the Workflow should be marked as "FAILED".
verifyProgramRuns(programId, "failed");
}
use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.
the class WorkflowStatsSLAHttpHandlerTest method testCompare.
@Test
public void testCompare() throws Exception {
deploy(WorkflowApp.class);
String workflowName = "FunWorkflow";
String mapreduceName = "ClassicWordCount";
String sparkName = "SparkWorkflowTest";
WorkflowId workflowProgram = WORKFLOW_APP.workflow(workflowName);
ProgramId mapreduceProgram = WORKFLOW_APP.mr(mapreduceName);
ProgramId sparkProgram = WORKFLOW_APP.spark(sparkName);
List<RunId> workflowRunIdList = setupRuns(workflowProgram, mapreduceProgram, sparkProgram, store, 2);
RunId workflowRun1 = workflowRunIdList.get(0);
RunId workflowRun2 = workflowRunIdList.get(1);
String request = String.format("%s/namespaces/%s/apps/%s/workflows/%s/runs/%s/compare?other-run-id=%s", Constants.Gateway.API_VERSION_3, Id.Namespace.DEFAULT.getId(), WorkflowApp.class.getSimpleName(), workflowProgram.getProgram(), workflowRun1.getId(), workflowRun2.getId());
HttpResponse response = doGet(request);
Collection<WorkflowStatsComparison.ProgramNodes> workflowStatistics = readResponse(response, new TypeToken<Collection<WorkflowStatsComparison.ProgramNodes>>() {
}.getType());
Assert.assertNotNull(workflowStatistics.iterator().next());
Assert.assertEquals(2, workflowStatistics.size());
for (WorkflowStatsComparison.ProgramNodes node : workflowStatistics) {
if (node.getProgramType() == ProgramType.MAPREDUCE) {
Assert.assertEquals(38L, (long) node.getWorkflowProgramDetailsList().get(0).getMetrics().get(TaskCounter.MAP_INPUT_RECORDS.name()));
}
}
}
use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.
the class WorkflowHttpHandlerTest method testWorkflowScopedArguments.
@Category(XSlowTests.class)
@Test
public void testWorkflowScopedArguments() throws Exception {
String workflowRunIdProperty = "workflowrunid";
HttpResponse response = deploy(WorkflowAppWithScopedParameters.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
ProgramId programId = Ids.namespace(TEST_NAMESPACE2).app(WorkflowAppWithScopedParameters.APP_NAME).workflow(WorkflowAppWithScopedParameters.ONE_WORKFLOW);
Map<String, String> runtimeArguments = Maps.newHashMap();
runtimeArguments.put("debug", "true");
runtimeArguments.put("mapreduce.*.debug", "false");
runtimeArguments.put("mapreduce.OneMR.debug", "true");
runtimeArguments.put("input.path", createInput("ProgramInput"));
runtimeArguments.put("mapreduce.OneMR.input.path", createInput("OneMRInput"));
runtimeArguments.put("mapreduce.OneMR.logical.start.time", "1234567890000");
runtimeArguments.put("mapreduce.AnotherMR.input.path", createInput("AnotherMRInput"));
runtimeArguments.put("spark.*.input.path", createInput("SparkInput"));
runtimeArguments.put("output.path", new File(tmpFolder.newFolder(), "ProgramOutput").getAbsolutePath());
runtimeArguments.put("mapreduce.OneMR.output.path", new File(tmpFolder.newFolder(), "OneMROutput").getAbsolutePath());
runtimeArguments.put("spark.AnotherSpark.output.path", new File(tmpFolder.newFolder(), "AnotherSparkOutput").getAbsolutePath());
runtimeArguments.put("mapreduce.*.processing.time", "1HR");
runtimeArguments.put("dataset.Purchase.cache.seconds", "30");
runtimeArguments.put("dataset.UserProfile.schema.property", "constant");
runtimeArguments.put("dataset.unknown.dataset", "false");
runtimeArguments.put("dataset.*.read.timeout", "60");
setAndTestRuntimeArgs(programId.toId(), runtimeArguments);
// Start the workflow
startProgram(programId.toId());
waitState(programId.toId(), ProgramStatus.RUNNING.name());
// Wait until we have a run record
verifyProgramRuns(programId.toId(), "running");
List<RunRecord> workflowHistoryRuns = getProgramRuns(programId.toId(), "running");
String workflowRunId = workflowHistoryRuns.get(0).getPid();
Id.Program mr1ProgramId = Id.Program.from(TEST_NAMESPACE2, WorkflowAppWithScopedParameters.APP_NAME, ProgramType.MAPREDUCE, WorkflowAppWithScopedParameters.ONE_MR);
waitState(mr1ProgramId, ProgramStatus.RUNNING.name());
List<RunRecord> oneMRHistoryRuns = getProgramRuns(mr1ProgramId, "running");
String expectedMessage = String.format("Cannot stop the program '%s' started by the Workflow run '%s'. " + "Please stop the Workflow.", new Id.Run(mr1ProgramId, oneMRHistoryRuns.get(0).getPid()), workflowRunId);
stopProgram(mr1ProgramId, oneMRHistoryRuns.get(0).getPid(), 400, expectedMessage);
verifyProgramRuns(programId.toId(), "completed");
workflowHistoryRuns = getProgramRuns(programId.toId(), "completed");
oneMRHistoryRuns = getProgramRuns(mr1ProgramId, "completed");
Id.Program mr2ProgramId = Id.Program.from(TEST_NAMESPACE2, WorkflowAppWithScopedParameters.APP_NAME, ProgramType.MAPREDUCE, WorkflowAppWithScopedParameters.ANOTHER_MR);
List<RunRecord> anotherMRHistoryRuns = getProgramRuns(mr2ProgramId, "completed");
Id.Program spark1ProgramId = Id.Program.from(TEST_NAMESPACE2, WorkflowAppWithScopedParameters.APP_NAME, ProgramType.SPARK, WorkflowAppWithScopedParameters.ONE_SPARK);
List<RunRecord> oneSparkHistoryRuns = getProgramRuns(spark1ProgramId, "completed");
Id.Program spark2ProgramId = Id.Program.from(TEST_NAMESPACE2, WorkflowAppWithScopedParameters.APP_NAME, ProgramType.SPARK, WorkflowAppWithScopedParameters.ANOTHER_SPARK);
List<RunRecord> anotherSparkHistoryRuns = getProgramRuns(spark2ProgramId, "completed");
Assert.assertEquals(1, workflowHistoryRuns.size());
Assert.assertEquals(1, oneMRHistoryRuns.size());
Assert.assertEquals(1, anotherMRHistoryRuns.size());
Assert.assertEquals(1, oneSparkHistoryRuns.size());
Assert.assertEquals(1, anotherSparkHistoryRuns.size());
Map<String, String> workflowRunRecordProperties = workflowHistoryRuns.get(0).getProperties();
Map<String, String> oneMRRunRecordProperties = oneMRHistoryRuns.get(0).getProperties();
Map<String, String> anotherMRRunRecordProperties = anotherMRHistoryRuns.get(0).getProperties();
Map<String, String> oneSparkRunRecordProperties = oneSparkHistoryRuns.get(0).getProperties();
Map<String, String> anotherSparkRunRecordProperties = anotherSparkHistoryRuns.get(0).getProperties();
Assert.assertNotNull(oneMRRunRecordProperties.get(workflowRunIdProperty));
Assert.assertEquals(workflowHistoryRuns.get(0).getPid(), oneMRRunRecordProperties.get(workflowRunIdProperty));
Assert.assertNotNull(anotherMRRunRecordProperties.get(workflowRunIdProperty));
Assert.assertEquals(workflowHistoryRuns.get(0).getPid(), anotherMRRunRecordProperties.get(workflowRunIdProperty));
Assert.assertNotNull(oneSparkRunRecordProperties.get(workflowRunIdProperty));
Assert.assertEquals(workflowHistoryRuns.get(0).getPid(), oneSparkRunRecordProperties.get(workflowRunIdProperty));
Assert.assertNotNull(anotherSparkRunRecordProperties.get(workflowRunIdProperty));
Assert.assertEquals(workflowHistoryRuns.get(0).getPid(), anotherSparkRunRecordProperties.get(workflowRunIdProperty));
Assert.assertEquals(workflowRunRecordProperties.get(WorkflowAppWithScopedParameters.ONE_MR), oneMRHistoryRuns.get(0).getPid());
Assert.assertEquals(workflowRunRecordProperties.get(WorkflowAppWithScopedParameters.ONE_SPARK), oneSparkHistoryRuns.get(0).getPid());
Assert.assertEquals(workflowRunRecordProperties.get(WorkflowAppWithScopedParameters.ANOTHER_MR), anotherMRHistoryRuns.get(0).getPid());
Assert.assertEquals(workflowRunRecordProperties.get(WorkflowAppWithScopedParameters.ANOTHER_SPARK), anotherSparkHistoryRuns.get(0).getPid());
// Get Workflow node states
Map<String, WorkflowNodeStateDetail> nodeStates = getWorkflowNodeStates(programId, workflowHistoryRuns.get(0).getPid());
Assert.assertNotNull(nodeStates);
Assert.assertEquals(5, nodeStates.size());
WorkflowNodeStateDetail mrNodeState = nodeStates.get(WorkflowAppWithScopedParameters.ONE_MR);
Assert.assertNotNull(mrNodeState);
Assert.assertEquals(WorkflowAppWithScopedParameters.ONE_MR, mrNodeState.getNodeId());
Assert.assertEquals(oneMRHistoryRuns.get(0).getPid(), mrNodeState.getRunId());
mrNodeState = nodeStates.get(WorkflowAppWithScopedParameters.ANOTHER_MR);
Assert.assertNotNull(mrNodeState);
Assert.assertEquals(WorkflowAppWithScopedParameters.ANOTHER_MR, mrNodeState.getNodeId());
Assert.assertEquals(anotherMRHistoryRuns.get(0).getPid(), mrNodeState.getRunId());
WorkflowNodeStateDetail sparkNodeState = nodeStates.get(WorkflowAppWithScopedParameters.ONE_SPARK);
Assert.assertNotNull(sparkNodeState);
Assert.assertEquals(WorkflowAppWithScopedParameters.ONE_SPARK, sparkNodeState.getNodeId());
Assert.assertEquals(oneSparkHistoryRuns.get(0).getPid(), sparkNodeState.getRunId());
sparkNodeState = nodeStates.get(WorkflowAppWithScopedParameters.ANOTHER_SPARK);
Assert.assertNotNull(sparkNodeState);
Assert.assertEquals(WorkflowAppWithScopedParameters.ANOTHER_SPARK, sparkNodeState.getNodeId());
Assert.assertEquals(anotherSparkHistoryRuns.get(0).getPid(), sparkNodeState.getRunId());
WorkflowNodeStateDetail oneActionNodeState = nodeStates.get(WorkflowAppWithScopedParameters.ONE_ACTION);
Assert.assertNotNull(oneActionNodeState);
Assert.assertEquals(WorkflowAppWithScopedParameters.ONE_ACTION, oneActionNodeState.getNodeId());
}
use of co.cask.cdap.proto.id.ProgramId in project cdap by caskdata.
the class WorkflowHttpHandlerTest method testStreamSizeSchedules.
@Test
public void testStreamSizeSchedules() throws Exception {
// Steps for the test:
// 1. Deploy the app
// 2. Verify the schedules
// 3. Ingest data in the stream
// 4. Verify the history after waiting a while
// 5. Suspend the schedule
// 6. Ingest data in the stream
// 7. Verify there are no runs after the suspend by looking at the history
// 8. Resume the schedule
// 9. Verify there are runs after the resume by looking at the history
String appName = "AppWithStreamSizeSchedule";
String sampleSchedule1 = "SampleSchedule1";
String sampleSchedule2 = "SampleSchedule2";
String workflowName = "SampleWorkflow";
String streamName = "stream";
Id.Program programId = Id.Program.from(TEST_NAMESPACE2, appName, ProgramType.WORKFLOW, workflowName);
StringBuilder longStringBuilder = new StringBuilder();
for (int i = 0; i < 10000; i++) {
longStringBuilder.append("dddddddddd");
}
String longString = longStringBuilder.toString();
// deploy app with schedule in namespace 2
HttpResponse response = deploy(AppWithStreamSizeSchedule.class, Constants.Gateway.API_VERSION_3_TOKEN, TEST_NAMESPACE2);
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
Assert.assertEquals(200, resumeSchedule(TEST_NAMESPACE2, appName, sampleSchedule1));
Assert.assertEquals(200, resumeSchedule(TEST_NAMESPACE2, appName, sampleSchedule2));
// get schedules
List<ScheduleDetail> schedules = getSchedules(TEST_NAMESPACE2, appName, workflowName);
Assert.assertEquals(2, schedules.size());
String scheduleName1 = schedules.get(0).getName();
String scheduleName2 = schedules.get(1).getName();
Assert.assertNotNull(scheduleName1);
Assert.assertFalse(scheduleName1.isEmpty());
// Change notification threshold for stream
response = doPut(String.format("/v3/namespaces/%s/streams/%s/properties", TEST_NAMESPACE2, streamName), "{'notification.threshold.mb': 1}");
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
response = doGet(String.format("/v3/namespaces/%s/streams/%s", TEST_NAMESPACE2, streamName));
String json = EntityUtils.toString(response.getEntity());
StreamProperties properties = new Gson().fromJson(json, StreamProperties.class);
Assert.assertEquals(1, properties.getNotificationThresholdMB().intValue());
// Ingest over 1MB of data in stream
for (int i = 0; i < 12; ++i) {
response = doPost(String.format("/v3/namespaces/%s/streams/%s", TEST_NAMESPACE2, streamName), longString);
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
}
// Only schedule 1 should get executed
verifyProgramRuns(programId, "completed");
//Check schedule status
assertSchedule(programId, scheduleName1, true, 30, TimeUnit.SECONDS);
assertSchedule(programId, scheduleName2, true, 30, TimeUnit.SECONDS);
Assert.assertEquals(200, suspendSchedule(TEST_NAMESPACE2, appName, scheduleName1));
Assert.assertEquals(200, suspendSchedule(TEST_NAMESPACE2, appName, scheduleName2));
//check paused state
assertSchedule(programId, scheduleName1, false, 30, TimeUnit.SECONDS);
assertSchedule(programId, scheduleName2, false, 30, TimeUnit.SECONDS);
int workflowRuns = getProgramRuns(programId, "completed").size();
// Should still be one
Assert.assertEquals(1, workflowRuns);
// Sleep for some time and verify there are no more scheduled jobs after the suspend.
for (int i = 0; i < 12; ++i) {
response = doPost(String.format("/v3/namespaces/%s/streams/%s", TEST_NAMESPACE2, streamName), longString);
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
}
TimeUnit.SECONDS.sleep(5);
int workflowRunsAfterSuspend = getProgramRuns(programId, "completed").size();
Assert.assertEquals(workflowRuns, workflowRunsAfterSuspend);
Assert.assertEquals(200, resumeSchedule(TEST_NAMESPACE2, appName, scheduleName1));
//check scheduled state
assertSchedule(programId, scheduleName1, true, 30, TimeUnit.SECONDS);
// an additional run should execute and complete after resuming the schedule
assertRunHistory(programId, "completed", 1 + workflowRunsAfterSuspend, 60, TimeUnit.SECONDS);
//Check status of a non existing schedule
try {
assertSchedule(programId, "invalid", true, 2, TimeUnit.SECONDS);
Assert.fail();
} catch (Exception e) {
// expected
}
Assert.assertEquals(200, suspendSchedule(TEST_NAMESPACE2, appName, scheduleName1));
//check paused state
assertSchedule(programId, scheduleName1, false, 30, TimeUnit.SECONDS);
//Schedule operations using invalid namespace
try {
assertSchedule(Id.Program.from(TEST_NAMESPACE1, appName, ProgramType.WORKFLOW, workflowName), scheduleName1, true, 2, TimeUnit.SECONDS);
Assert.fail();
} catch (Exception e) {
// expected
}
Assert.assertEquals(404, suspendSchedule(TEST_NAMESPACE1, appName, scheduleName1));
Assert.assertEquals(404, resumeSchedule(TEST_NAMESPACE1, appName, scheduleName1));
// Wait until any running jobs just before suspend call completes.
TimeUnit.SECONDS.sleep(2);
}
Aggregations