use of io.cdap.cdap.api.artifact.ArtifactId in project cdap by caskdata.
the class RunnableTaskLauncherTest method testRunnableTaskRequestJsonConversion.
@Test
public void testRunnableTaskRequestJsonConversion() {
ArtifactId artifactId = new ArtifactId("test-artifact", new ArtifactVersion("1.0"), ArtifactScope.SYSTEM);
RunnableTaskRequest request = RunnableTaskRequest.getBuilder("taskClassName").withParam("param").withNamespace("n1").withArtifact(artifactId).build();
Gson gson = new Gson();
String requestJson = gson.toJson(request);
RunnableTaskRequest requestFromJson = gson.fromJson(requestJson, RunnableTaskRequest.class);
Assert.assertEquals(request.getClassName(), requestFromJson.getClassName());
Assert.assertEquals(request.getNamespace(), requestFromJson.getNamespace());
Assert.assertEquals(request.getParam(), requestFromJson.getParam());
Assert.assertEquals(request.getArtifactId(), requestFromJson.getArtifactId());
}
use of io.cdap.cdap.api.artifact.ArtifactId in project cdap by caskdata.
the class DefaultStoreTest method testDeleteSuspendedWorkflow.
@Test
public void testDeleteSuspendedWorkflow() {
NamespaceId namespaceId = new NamespaceId("namespace1");
// Test delete application
ApplicationId appId1 = namespaceId.app("app1");
ProgramId programId1 = appId1.workflow("pgm1");
ArtifactId artifactId = namespaceId.artifact("testArtifact", "1.0").toApiArtifactId();
RunId run1 = RunIds.generate();
setStartAndRunning(programId1.run(run1.getId()), artifactId);
store.setSuspend(programId1.run(run1.getId()), AppFabricTestHelper.createSourceId(++sourceId), -1);
store.removeApplication(appId1);
Assert.assertTrue(store.getRuns(programId1, ProgramRunStatus.ALL, 0, Long.MAX_VALUE, Integer.MAX_VALUE).isEmpty());
// Test delete namespace
ProgramId programId2 = namespaceId.app("app2").workflow("pgm2");
RunId run2 = RunIds.generate();
setStartAndRunning(programId2.run(run2.getId()), artifactId);
store.setSuspend(programId2.run(run2.getId()), AppFabricTestHelper.createSourceId(++sourceId), -1);
store.removeAll(namespaceId);
nsStore.delete(namespaceId);
Assert.assertTrue(store.getRuns(programId2, ProgramRunStatus.ALL, 0, Long.MAX_VALUE, Integer.MAX_VALUE).isEmpty());
}
use of io.cdap.cdap.api.artifact.ArtifactId in project cdap by caskdata.
the class DefaultStoreTest method testProgramRunCount.
@Test
public void testProgramRunCount() {
ApplicationSpecification spec = Specifications.from(new AllProgramsApp());
ApplicationId appId = NamespaceId.DEFAULT.app(spec.getName());
ArtifactId testArtifact = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
ProgramId workflowId = appId.workflow(AllProgramsApp.NoOpWorkflow.NAME);
ProgramId serviceId = appId.service(AllProgramsApp.NoOpService.NAME);
ProgramId nonExistingAppProgramId = NamespaceId.DEFAULT.app("nonExisting").workflow("test");
ProgramId nonExistingProgramId = appId.workflow("nonExisting");
// add the application
store.addApplication(appId, spec);
// add some run records to workflow and service
for (int i = 0; i < 5; i++) {
setStart(workflowId.run(RunIds.generate()), Collections.emptyMap(), Collections.emptyMap(), testArtifact);
setStart(serviceId.run(RunIds.generate()), Collections.emptyMap(), Collections.emptyMap(), testArtifact);
}
List<RunCountResult> result = store.getProgramRunCounts(ImmutableList.of(workflowId, serviceId, nonExistingAppProgramId, nonExistingProgramId));
// compare the result
Assert.assertEquals(4, result.size());
for (RunCountResult runCountResult : result) {
ProgramId programId = runCountResult.getProgramId();
Long count = runCountResult.getCount();
if (programId.equals(nonExistingAppProgramId) || programId.equals(nonExistingProgramId)) {
Assert.assertNull(count);
Assert.assertTrue(runCountResult.getException() instanceof NotFoundException);
} else {
Assert.assertNotNull(count);
Assert.assertEquals(5L, count.longValue());
}
}
// remove the app should remove all run count
store.removeApplication(appId);
for (RunCountResult runCountResult : store.getProgramRunCounts(ImmutableList.of(workflowId, serviceId))) {
Assert.assertNull(runCountResult.getCount());
Assert.assertTrue(runCountResult.getException() instanceof NotFoundException);
}
}
use of io.cdap.cdap.api.artifact.ArtifactId in project cdap by caskdata.
the class DefaultStoreTest method testLogProgramRunHistory.
@Test
public void testLogProgramRunHistory() {
Map<String, String> noRuntimeArgsProps = ImmutableMap.of("runtimeArgs", GSON.toJson(ImmutableMap.<String, String>of()));
// record finished Workflow
ProgramId programId = new ProgramId("account1", "application1", ProgramType.WORKFLOW, "wf1");
long now = System.currentTimeMillis();
long startTimeSecs = TimeUnit.MILLISECONDS.toSeconds(now);
RunId run1 = RunIds.generate(now - 20000);
ArtifactId artifactId = programId.getNamespaceId().artifact("testArtifact", "1.0").toApiArtifactId();
setStartAndRunning(programId.run(run1.getId()), artifactId);
store.setStop(programId.run(run1.getId()), startTimeSecs - 10, ProgramController.State.ERROR.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId));
// record another finished Workflow
RunId run2 = RunIds.generate(now - 10000);
setStartAndRunning(programId.run(run2.getId()), artifactId);
store.setStop(programId.run(run2.getId()), startTimeSecs - 5, ProgramController.State.COMPLETED.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId));
// record a suspended Workflow
RunId run21 = RunIds.generate(now - 7500);
setStartAndRunning(programId.run(run21.getId()), artifactId);
store.setSuspend(programId.run(run21.getId()), AppFabricTestHelper.createSourceId(++sourceId), -1);
// record not finished Workflow
RunId run3 = RunIds.generate(now);
setStartAndRunning(programId.run(run3.getId()), artifactId);
// For a RunRecordDetail that has not yet been completed, getStopTs should return null
RunRecordDetail runRecord = store.getRun(programId.run(run3.getId()));
Assert.assertNotNull(runRecord);
Assert.assertNull(runRecord.getStopTs());
// record run of different program
ProgramId programId2 = new ProgramId("account1", "application1", ProgramType.WORKFLOW, "wf2");
RunId run4 = RunIds.generate(now - 5000);
setStartAndRunning(programId2.run(run4.getId()), artifactId);
store.setStop(programId2.run(run4.getId()), startTimeSecs - 4, ProgramController.State.COMPLETED.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId));
// record for different account
setStartAndRunning(new ProgramId("account2", "application1", ProgramType.WORKFLOW, "wf1").run(run3.getId()), artifactId);
// we should probably be better with "get" method in DefaultStore interface to do that, but we don't have one
Map<ProgramRunId, RunRecordDetail> successHistorymap = store.getRuns(programId, ProgramRunStatus.COMPLETED, 0, Long.MAX_VALUE, Integer.MAX_VALUE);
Map<ProgramRunId, RunRecordDetail> failureHistorymap = store.getRuns(programId, ProgramRunStatus.FAILED, startTimeSecs - 20, startTimeSecs - 10, Integer.MAX_VALUE);
Assert.assertEquals(failureHistorymap, store.getRuns(programId, ProgramRunStatus.FAILED, 0, Long.MAX_VALUE, Integer.MAX_VALUE));
Map<ProgramRunId, RunRecordDetail> suspendedHistorymap = store.getRuns(programId, ProgramRunStatus.SUSPENDED, startTimeSecs - 20, startTimeSecs, Integer.MAX_VALUE);
// only finished + succeeded runs should be returned
Assert.assertEquals(1, successHistorymap.size());
// only finished + failed runs should be returned
Assert.assertEquals(1, failureHistorymap.size());
// only suspended runs should be returned
Assert.assertEquals(1, suspendedHistorymap.size());
// records should be sorted by start time latest to earliest
RunRecordDetail run = successHistorymap.values().iterator().next();
Assert.assertEquals(startTimeSecs - 10, run.getStartTs());
Assert.assertEquals(Long.valueOf(startTimeSecs - 5), run.getStopTs());
Assert.assertEquals(ProgramController.State.COMPLETED.getRunStatus(), run.getStatus());
run = failureHistorymap.values().iterator().next();
Assert.assertEquals(startTimeSecs - 20, run.getStartTs());
Assert.assertEquals(Long.valueOf(startTimeSecs - 10), run.getStopTs());
Assert.assertEquals(ProgramController.State.ERROR.getRunStatus(), run.getStatus());
run = suspendedHistorymap.values().iterator().next();
Assert.assertEquals(run21.getId(), run.getPid());
Assert.assertEquals(ProgramController.State.SUSPENDED.getRunStatus(), run.getStatus());
// Assert all history
Map<ProgramRunId, RunRecordDetail> allHistorymap = store.getRuns(programId, ProgramRunStatus.ALL, startTimeSecs - 20, startTimeSecs + 1, Integer.MAX_VALUE);
Assert.assertEquals(allHistorymap.toString(), 4, allHistorymap.size());
// Assert running programs
Map<ProgramRunId, RunRecordDetail> runningHistorymap = store.getRuns(programId, ProgramRunStatus.RUNNING, startTimeSecs, startTimeSecs + 1, 100);
Assert.assertEquals(1, runningHistorymap.size());
Assert.assertEquals(runningHistorymap, store.getRuns(programId, ProgramRunStatus.RUNNING, 0, Long.MAX_VALUE, 100));
// Get a run record for running program
RunRecordDetail expectedRunning = runningHistorymap.values().iterator().next();
Assert.assertNotNull(expectedRunning);
RunRecordDetail actualRunning = store.getRun(programId.run(expectedRunning.getPid()));
Assert.assertEquals(expectedRunning, actualRunning);
// Get a run record for completed run
RunRecordDetail expectedCompleted = successHistorymap.values().iterator().next();
Assert.assertNotNull(expectedCompleted);
RunRecordDetail actualCompleted = store.getRun(programId.run(expectedCompleted.getPid()));
Assert.assertEquals(expectedCompleted, actualCompleted);
// Get a run record for suspended run
RunRecordDetail expectedSuspended = suspendedHistorymap.values().iterator().next();
Assert.assertNotNull(expectedSuspended);
RunRecordDetail actualSuspended = store.getRun(programId.run(expectedSuspended.getPid()));
Assert.assertEquals(expectedSuspended, actualSuspended);
ProgramRunCluster emptyCluster = new ProgramRunCluster(ProgramRunClusterStatus.PROVISIONED, null, 0);
// Record workflow that starts but encounters error before it runs
RunId run7 = RunIds.generate(now);
Map<String, String> emptyArgs = ImmutableMap.of();
setStart(programId.run(run7.getId()), emptyArgs, emptyArgs, artifactId);
store.setStop(programId.run(run7.getId()), startTimeSecs + 1, ProgramController.State.ERROR.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId));
RunRecordDetail expectedRunRecord7 = RunRecordDetail.builder().setProgramRunId(programId.run(run7)).setStartTime(startTimeSecs).setStopTime(startTimeSecs + 1).setStatus(ProgramRunStatus.FAILED).setProperties(noRuntimeArgsProps).setCluster(emptyCluster).setArtifactId(artifactId).setSourceId(AppFabricTestHelper.createSourceId(sourceId)).build();
RunRecordDetail actualRecord7 = store.getRun(programId.run(run7.getId()));
Assert.assertEquals(expectedRunRecord7, actualRecord7);
// Record workflow that starts and suspends before it runs
RunId run8 = RunIds.generate(now);
setStart(programId.run(run8.getId()), emptyArgs, emptyArgs, artifactId);
store.setSuspend(programId.run(run8.getId()), AppFabricTestHelper.createSourceId(++sourceId), -1);
RunRecordDetail expectedRunRecord8 = RunRecordDetail.builder().setProgramRunId(programId.run(run8)).setStartTime(startTimeSecs).setStatus(ProgramRunStatus.SUSPENDED).setProperties(noRuntimeArgsProps).setCluster(emptyCluster).setArtifactId(artifactId).setSourceId(AppFabricTestHelper.createSourceId(sourceId)).build();
RunRecordDetail actualRecord8 = store.getRun(programId.run(run8.getId()));
Assert.assertEquals(expectedRunRecord8, actualRecord8);
// Record workflow that is killed while suspended
RunId run9 = RunIds.generate(now);
setStartAndRunning(programId.run(run9.getId()), artifactId);
store.setSuspend(programId.run(run9.getId()), AppFabricTestHelper.createSourceId(++sourceId), -1);
store.setStop(programId.run(run9.getId()), startTimeSecs + 5, ProgramRunStatus.KILLED, AppFabricTestHelper.createSourceId(++sourceId));
RunRecordDetail expectedRunRecord9 = RunRecordDetail.builder().setProgramRunId(programId.run(run9)).setStartTime(startTimeSecs).setRunTime(startTimeSecs + 1).setStopTime(startTimeSecs + 5).setStatus(ProgramRunStatus.KILLED).setProperties(noRuntimeArgsProps).setCluster(emptyCluster).setArtifactId(artifactId).setSourceId(AppFabricTestHelper.createSourceId(sourceId)).build();
RunRecordDetail actualRecord9 = store.getRun(programId.run(run9.getId()));
Assert.assertEquals(expectedRunRecord9, actualRecord9);
// Non-existent run record should give null
Assert.assertNull(store.getRun(programId.run(UUID.randomUUID().toString())));
// Searching for history in wrong time range should give us no results
Assert.assertTrue(store.getRuns(programId, ProgramRunStatus.COMPLETED, startTimeSecs - 5000, startTimeSecs - 2000, Integer.MAX_VALUE).isEmpty());
Assert.assertTrue(store.getRuns(programId, ProgramRunStatus.ALL, startTimeSecs - 5000, startTimeSecs - 2000, Integer.MAX_VALUE).isEmpty());
}
use of io.cdap.cdap.api.artifact.ArtifactId in project cdap by caskdata.
the class DefaultStoreTest method testWorkflowNodeState.
@Test
public void testWorkflowNodeState() {
String namespaceName = "namespace1";
String appName = "app1";
String workflowName = "workflow1";
String mapReduceName = "mapReduce1";
String sparkName = "spark1";
ApplicationId appId = Ids.namespace(namespaceName).app(appName);
ProgramId mapReduceProgram = appId.mr(mapReduceName);
ProgramId sparkProgram = appId.spark(sparkName);
long currentTime = System.currentTimeMillis();
String workflowRunId = RunIds.generate(currentTime).getId();
ProgramRunId workflowRun = appId.workflow(workflowName).run(workflowRunId);
ArtifactId artifactId = appId.getParent().artifact("testArtifact", "1.0").toApiArtifactId();
// start Workflow
setStartAndRunning(workflowRun, artifactId);
// start MapReduce as a part of Workflow
Map<String, String> systemArgs = ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NODE_ID, mapReduceName, ProgramOptionConstants.WORKFLOW_NAME, workflowName, ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId);
RunId mapReduceRunId = RunIds.generate(currentTime + 10);
setStartAndRunning(mapReduceProgram.run(mapReduceRunId.getId()), ImmutableMap.of(), systemArgs, artifactId);
// stop the MapReduce program
store.setStop(mapReduceProgram.run(mapReduceRunId.getId()), currentTime + 50, ProgramRunStatus.COMPLETED, AppFabricTestHelper.createSourceId(++sourceId));
// start Spark program as a part of Workflow
systemArgs = ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NODE_ID, sparkName, ProgramOptionConstants.WORKFLOW_NAME, workflowName, ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId);
RunId sparkRunId = RunIds.generate(currentTime + 60);
setStartAndRunning(sparkProgram.run(sparkRunId.getId()), ImmutableMap.of(), systemArgs, artifactId);
// stop the Spark program with failure
NullPointerException npe = new NullPointerException("dataset not found");
IllegalArgumentException iae = new IllegalArgumentException("illegal argument", npe);
store.setStop(sparkProgram.run(sparkRunId.getId()), currentTime + 100, ProgramRunStatus.FAILED, new BasicThrowable(iae), AppFabricTestHelper.createSourceId(++sourceId));
// stop Workflow
store.setStop(workflowRun, currentTime + 110, ProgramRunStatus.FAILED, AppFabricTestHelper.createSourceId(++sourceId));
List<WorkflowNodeStateDetail> nodeStateDetails = store.getWorkflowNodeStates(workflowRun);
Map<String, WorkflowNodeStateDetail> workflowNodeStates = new HashMap<>();
for (WorkflowNodeStateDetail nodeStateDetail : nodeStateDetails) {
workflowNodeStates.put(nodeStateDetail.getNodeId(), nodeStateDetail);
}
Assert.assertEquals(2, workflowNodeStates.size());
WorkflowNodeStateDetail nodeStateDetail = workflowNodeStates.get(mapReduceName);
Assert.assertEquals(mapReduceName, nodeStateDetail.getNodeId());
Assert.assertEquals(NodeStatus.COMPLETED, nodeStateDetail.getNodeStatus());
Assert.assertEquals(mapReduceRunId.getId(), nodeStateDetail.getRunId());
Assert.assertNull(nodeStateDetail.getFailureCause());
nodeStateDetail = workflowNodeStates.get(sparkName);
Assert.assertEquals(sparkName, nodeStateDetail.getNodeId());
Assert.assertEquals(NodeStatus.FAILED, nodeStateDetail.getNodeStatus());
Assert.assertEquals(sparkRunId.getId(), nodeStateDetail.getRunId());
BasicThrowable failureCause = nodeStateDetail.getFailureCause();
Assert.assertNotNull(failureCause);
Assert.assertEquals("illegal argument", failureCause.getMessage());
Assert.assertEquals(IllegalArgumentException.class.getName(), failureCause.getClassName());
failureCause = failureCause.getCause();
Assert.assertNotNull(failureCause);
Assert.assertEquals("dataset not found", failureCause.getMessage());
Assert.assertEquals(NullPointerException.class.getName(), failureCause.getClassName());
Assert.assertNull(failureCause.getCause());
}
Aggregations