use of io.cdap.cdap.api.workflow.ScheduleProgramInfo in project cdap by caskdata.
the class ProgramLifecycleHttpHandlerTest method testAddSchedule.
private void testAddSchedule(String scheduleName) throws Exception {
String partitionScheduleName = scheduleName + "Partition";
String orScheduleName = scheduleName + "Or";
ProtoTrigger.TimeTrigger protoTime = new ProtoTrigger.TimeTrigger("0 * * * ?");
ProtoTrigger.PartitionTrigger protoPartition = new ProtoTrigger.PartitionTrigger(NamespaceId.DEFAULT.dataset("data"), 5);
ProtoTrigger.OrTrigger protoOr = ProtoTrigger.or(protoTime, protoPartition);
String description = "Something";
ScheduleProgramInfo programInfo = new ScheduleProgramInfo(SchedulableProgramType.WORKFLOW, AppWithSchedule.WORKFLOW_NAME);
ImmutableMap<String, String> properties = ImmutableMap.of("a", "b", "c", "d");
TimeTrigger timeTrigger = new TimeTrigger("0 * * * ?");
ScheduleDetail timeDetail = new ScheduleDetail(TEST_NAMESPACE1, AppWithSchedule.NAME, ApplicationId.DEFAULT_VERSION, scheduleName, description, programInfo, properties, timeTrigger, Collections.emptyList(), Schedulers.JOB_QUEUE_TIMEOUT_MILLIS, null, null);
PartitionTrigger partitionTrigger = new PartitionTrigger(protoPartition.getDataset(), protoPartition.getNumPartitions());
ScheduleDetail expectedPartitionDetail = new ScheduleDetail(TEST_NAMESPACE1, AppWithSchedule.NAME, ApplicationId.DEFAULT_VERSION, partitionScheduleName, description, programInfo, properties, partitionTrigger, Collections.emptyList(), Schedulers.JOB_QUEUE_TIMEOUT_MILLIS, null, null);
ScheduleDetail requestPartitionDetail = new ScheduleDetail(TEST_NAMESPACE1, AppWithSchedule.NAME, ApplicationId.DEFAULT_VERSION, partitionScheduleName, description, programInfo, properties, protoPartition, Collections.emptyList(), Schedulers.JOB_QUEUE_TIMEOUT_MILLIS, null, null);
ScheduleDetail expectedOrDetail = new ScheduleDetail(TEST_NAMESPACE1, AppWithSchedule.NAME, ApplicationId.DEFAULT_VERSION, orScheduleName, description, programInfo, properties, new OrTrigger(timeTrigger, partitionTrigger), Collections.emptyList(), Schedulers.JOB_QUEUE_TIMEOUT_MILLIS, null, null);
ScheduleDetail requestOrDetail = new ScheduleDetail(TEST_NAMESPACE1, AppWithSchedule.NAME, ApplicationId.DEFAULT_VERSION, orScheduleName, description, programInfo, properties, protoOr, Collections.emptyList(), Schedulers.JOB_QUEUE_TIMEOUT_MILLIS, null, null);
// trying to add the schedule with different name in path param than schedule spec should fail
HttpResponse response = addSchedule(TEST_NAMESPACE1, AppWithSchedule.NAME, null, "differentName", timeDetail);
Assert.assertEquals(HttpResponseStatus.BAD_REQUEST.code(), response.getResponseCode());
// adding a schedule to a non-existing app should fail
response = addSchedule(TEST_NAMESPACE1, "nonExistingApp", null, scheduleName, timeDetail);
Assert.assertEquals(HttpResponseStatus.NOT_FOUND.code(), response.getResponseCode());
// adding a schedule to invalid type of program type should fail
ScheduleDetail invalidScheduleDetail = new ScheduleDetail(scheduleName, "Something", new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, AppWithSchedule.MAPREDUCE), properties, protoTime, Collections.emptyList(), TimeUnit.MINUTES.toMillis(1));
response = addSchedule(TEST_NAMESPACE1, AppWithSchedule.NAME, null, scheduleName, invalidScheduleDetail);
Assert.assertEquals(HttpResponseStatus.BAD_REQUEST.code(), response.getResponseCode());
// adding a schedule for a program that does not exist
ScheduleDetail nonExistingDetail = new ScheduleDetail(TEST_NAMESPACE1, AppWithSchedule.NAME, ApplicationId.DEFAULT_VERSION, scheduleName, description, new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "nope"), properties, timeTrigger, Collections.emptyList(), Schedulers.JOB_QUEUE_TIMEOUT_MILLIS, null, null);
response = addSchedule(TEST_NAMESPACE1, AppWithSchedule.NAME, null, scheduleName, nonExistingDetail);
Assert.assertEquals(HttpResponseStatus.NOT_FOUND.code(), response.getResponseCode());
// test adding a schedule
response = addSchedule(TEST_NAMESPACE1, AppWithSchedule.NAME, null, scheduleName, timeDetail);
Assert.assertEquals(HttpResponseStatus.OK.code(), response.getResponseCode());
response = addSchedule(TEST_NAMESPACE1, AppWithSchedule.NAME, null, partitionScheduleName, requestPartitionDetail);
Assert.assertEquals(HttpResponseStatus.OK.code(), response.getResponseCode());
response = addSchedule(TEST_NAMESPACE1, AppWithSchedule.NAME, null, orScheduleName, requestOrDetail);
Assert.assertEquals(HttpResponseStatus.OK.code(), response.getResponseCode());
List<ScheduleDetail> schedules = getSchedules(TEST_NAMESPACE1, AppWithSchedule.NAME, AppWithSchedule.WORKFLOW_NAME);
Assert.assertEquals(4, schedules.size());
Assert.assertEquals(timeDetail, schedules.get(1));
Assert.assertEquals(expectedOrDetail, schedules.get(2));
Assert.assertEquals(expectedPartitionDetail, schedules.get(3));
List<ScheduleDetail> schedulesForApp = listSchedules(TEST_NAMESPACE1, AppWithSchedule.NAME, null);
Assert.assertEquals(schedules, schedulesForApp);
// trying to add ScheduleDetail of the same schedule again should fail with AlreadyExistsException
response = addSchedule(TEST_NAMESPACE1, AppWithSchedule.NAME, null, scheduleName, timeDetail);
Assert.assertEquals(HttpResponseStatus.CONFLICT.code(), response.getResponseCode());
// although we should be able to add schedule to a different version of the app
response = addSchedule(TEST_NAMESPACE1, AppWithSchedule.NAME, VERSION2, scheduleName, timeDetail);
Assert.assertEquals(HttpResponseStatus.OK.code(), response.getResponseCode());
// this should not have affected the schedules of the default version
List<ScheduleDetail> scheds = getSchedules(TEST_NAMESPACE1, AppWithSchedule.NAME, AppWithSchedule.WORKFLOW_NAME);
Assert.assertEquals(schedules, scheds);
// there should be two schedules now for version 2
List<ScheduleDetail> schedules2 = getSchedules(TEST_NAMESPACE1, AppWithSchedule.NAME, VERSION2, AppWithSchedule.WORKFLOW_NAME);
Assert.assertEquals(2, schedules2.size());
Assert.assertEquals(timeDetail, schedules2.get(1));
List<ScheduleDetail> schedulesForApp2 = listSchedules(TEST_NAMESPACE1, AppWithSchedule.NAME, VERSION2);
Assert.assertEquals(schedules2, schedulesForApp2);
// Add a schedule with no schedule name in spec
ScheduleDetail detail2 = new ScheduleDetail(TEST_NAMESPACE1, AppWithSchedule.NAME, VERSION2, null, "Something 2", programInfo, properties, new TimeTrigger("0 * * * ?"), Collections.emptyList(), TimeUnit.HOURS.toMillis(6), null, null);
response = addSchedule(TEST_NAMESPACE1, AppWithSchedule.NAME, VERSION2, "schedule-100", detail2);
Assert.assertEquals(HttpResponseStatus.OK.code(), response.getResponseCode());
ScheduleDetail detail100 = getSchedule(TEST_NAMESPACE1, AppWithSchedule.NAME, VERSION2, "schedule-100");
Assert.assertEquals("schedule-100", detail100.getName());
Assert.assertEquals(detail2.getTimeoutMillis(), detail100.getTimeoutMillis());
}
use of io.cdap.cdap.api.workflow.ScheduleProgramInfo in project cdap by caskdata.
the class WorkflowVerificationTest method verifyWorkflowWithLocalDatasetSpecification.
private void verifyWorkflowWithLocalDatasetSpecification(ApplicationSpecification appSpec) {
WorkflowSpecification spec = appSpec.getWorkflows().get("WorkflowWithLocalDatasets");
List<WorkflowNode> nodes = spec.getNodes();
Assert.assertTrue(nodes.size() == 2);
WorkflowNode node = nodes.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
WorkflowActionNode actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR1")));
node = nodes.get(1);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP1")));
Map<String, DatasetCreationSpec> localDatasetSpecs = spec.getLocalDatasetSpecs();
Assert.assertEquals(5, localDatasetSpecs.size());
DatasetCreationSpec datasetCreationSpec = localDatasetSpecs.get("mytable");
Assert.assertEquals(Table.class.getName(), datasetCreationSpec.getTypeName());
Assert.assertEquals(0, datasetCreationSpec.getProperties().getProperties().size());
datasetCreationSpec = localDatasetSpecs.get("myfile");
Assert.assertEquals(FileSet.class.getName(), datasetCreationSpec.getTypeName());
Assert.assertEquals(0, datasetCreationSpec.getProperties().getProperties().size());
datasetCreationSpec = localDatasetSpecs.get("myfile_with_properties");
Assert.assertEquals(FileSet.class.getName(), datasetCreationSpec.getTypeName());
Assert.assertEquals("prop_value", datasetCreationSpec.getProperties().getProperties().get("prop_key"));
datasetCreationSpec = localDatasetSpecs.get("mytablefromtype");
Assert.assertEquals(Table.class.getName(), datasetCreationSpec.getTypeName());
Assert.assertEquals(0, datasetCreationSpec.getProperties().getProperties().size());
datasetCreationSpec = localDatasetSpecs.get("myfilefromtype");
Assert.assertEquals(FileSet.class.getName(), datasetCreationSpec.getTypeName());
Assert.assertEquals("another_prop_value", datasetCreationSpec.getProperties().getProperties().get("another_prop_key"));
// Check if the application specification has correct modules
Map<String, String> datasetModules = appSpec.getDatasetModules();
Assert.assertEquals(2, datasetModules.size());
Assert.assertTrue(datasetModules.containsKey(FileSet.class.getName()));
Assert.assertTrue(datasetModules.containsKey(Table.class.getName()));
}
use of io.cdap.cdap.api.workflow.ScheduleProgramInfo in project cdap by caskdata.
the class WorkflowVerificationTest method verifyAnotherGoodWorkflowSpecification.
private void verifyAnotherGoodWorkflowSpecification(ApplicationSpecification appSpec) {
WorkflowSpecification spec = appSpec.getWorkflows().get("AnotherGoodWorkflow");
List<WorkflowNode> nodes = spec.getNodes();
Assert.assertTrue(nodes.size() == 4);
WorkflowNode node = nodes.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
WorkflowActionNode actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR1")));
node = nodes.get(1);
Assert.assertTrue(node.getType() == WorkflowNodeType.FORK);
WorkflowForkNode fork = (WorkflowForkNode) node;
Assert.assertTrue(fork.getBranches().size() == 2);
List<WorkflowNode> forkBranch1 = fork.getBranches().get(0);
Assert.assertTrue(forkBranch1.size() == 3);
node = forkBranch1.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR2")));
node = forkBranch1.get(1);
Assert.assertTrue(node.getType() == WorkflowNodeType.CONDITION);
WorkflowConditionNode condition = (WorkflowConditionNode) node;
Assert.assertTrue(condition.getPredicateClassName().contains("MyVerificationPredicate"));
List<WorkflowNode> ifNodes = condition.getIfBranch();
Assert.assertTrue(ifNodes.size() == 2);
List<WorkflowNode> elseNodes = condition.getElseBranch();
Assert.assertTrue(elseNodes.size() == 2);
node = ifNodes.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR3")));
node = ifNodes.get(1);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR4")));
node = elseNodes.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR5")));
node = elseNodes.get(1);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR6")));
node = forkBranch1.get(2);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR7")));
List<WorkflowNode> forkBranch2 = fork.getBranches().get(1);
Assert.assertTrue(forkBranch2.size() == 1);
node = forkBranch2.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "MR8")));
node = nodes.get(2);
Assert.assertTrue(node.getType() == WorkflowNodeType.CONDITION);
ifNodes = ((WorkflowConditionNode) node).getIfBranch();
elseNodes = ((WorkflowConditionNode) node).getElseBranch();
Assert.assertTrue(ifNodes.size() == 2);
node = ifNodes.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP1")));
node = ifNodes.get(1);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP2")));
Assert.assertTrue(elseNodes.size() == 3);
node = elseNodes.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP3")));
node = elseNodes.get(1);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP4")));
node = elseNodes.get(2);
Assert.assertTrue(node.getType() == WorkflowNodeType.FORK);
WorkflowForkNode anotherFork = (WorkflowForkNode) node;
Assert.assertTrue(anotherFork.getBranches().size() == 2);
List<WorkflowNode> anotherForkBranch1 = anotherFork.getBranches().get(0);
Assert.assertTrue(anotherForkBranch1.size() == 1);
List<WorkflowNode> anotherForkBranch2 = anotherFork.getBranches().get(1);
Assert.assertTrue(anotherForkBranch2.size() == 1);
node = anotherForkBranch1.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP5")));
node = anotherForkBranch2.get(0);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP6")));
node = nodes.get(3);
Assert.assertTrue(node.getType() == WorkflowNodeType.ACTION);
actionNode = (WorkflowActionNode) node;
Assert.assertTrue(actionNode.getProgram().equals(new ScheduleProgramInfo(SchedulableProgramType.SPARK, "SP7")));
}
use of io.cdap.cdap.api.workflow.ScheduleProgramInfo in project cdap by caskdata.
the class ScheduledRunTimeTest method testGetNextRun.
@Test
public void testGetNextRun() throws Exception {
ApplicationId appId = NamespaceId.DEFAULT.app("test");
deploy(appId, new AppRequest<>(new ArtifactSummary(ARTIFACT_ID.getName(), ARTIFACT_ID.getVersion().getVersion())));
String scheduleName = "schedule1";
// Add a schedule. Use a constraint to make it not going to be executed
ProgramId programId = appId.workflow(WorkflowApp.FunWorkflow.NAME);
Constraint constraint = new DelayConstraint(1, TimeUnit.HOURS);
ScheduleProgramInfo scheduleProgramInfo = new ScheduleProgramInfo(programId.getType().getSchedulableType(), programId.getProgram());
addSchedule(appId.getNamespace(), appId.getApplication(), appId.getVersion(), scheduleName, new ScheduleDetail(scheduleName, null, scheduleProgramInfo, null, new TimeTrigger("0 0 * * * "), Collections.singletonList(constraint), null));
long now = System.currentTimeMillis();
HttpResponse response = enableSchedule(programId.getNamespace(), programId.getApplication(), programId.getVersion(), scheduleName);
Assert.assertEquals(200, response.getResponseCode());
// Get the next run time
List<ScheduledRuntime> scheduledRunTimes = getScheduledRunTimes(programId, true);
Assert.assertEquals(1, scheduledRunTimes.size());
long nextTime = scheduledRunTimes.get(0).getTime();
Assert.assertTrue(nextTime >= now);
}
use of io.cdap.cdap.api.workflow.ScheduleProgramInfo in project cdap by caskdata.
the class ScheduledRunTimeTest method testBatchGetNextRun.
@Test
public void testBatchGetNextRun() throws Exception {
// deploys 5 apps and create schedules for each of them
long now = System.currentTimeMillis();
List<ProgramId> programIds = new ArrayList<>();
// Use a constraint to make it not going to be executed
Constraint constraint = new DelayConstraint(1, TimeUnit.HOURS);
for (int i = 0; i < 5; i++) {
ApplicationId appId = NamespaceId.DEFAULT.app("test" + i);
deploy(appId, new AppRequest<>(new ArtifactSummary(ARTIFACT_ID.getName(), ARTIFACT_ID.getVersion().getVersion())));
String scheduleName = "schedule" + i;
// Add a schedule
ProgramId programId = appId.workflow(WorkflowApp.FunWorkflow.NAME);
programIds.add(programId);
ScheduleProgramInfo scheduleProgramInfo = new ScheduleProgramInfo(programId.getType().getSchedulableType(), programId.getProgram());
addSchedule(appId.getNamespace(), appId.getApplication(), appId.getVersion(), scheduleName, new ScheduleDetail(scheduleName, null, scheduleProgramInfo, null, new TimeTrigger("0 0 * * * "), Collections.singletonList(constraint), null));
HttpResponse response = enableSchedule(programId.getNamespace(), programId.getApplication(), programId.getVersion(), scheduleName);
Assert.assertEquals(200, response.getResponseCode());
}
// Add programs that the app or the program doesn't exist
programIds.add(NamespaceId.DEFAULT.app("not-exist").workflow("not-exist"));
programIds.add(NamespaceId.DEFAULT.app("test1").workflow("not-exist"));
List<BatchProgramSchedule> schedules = getScheduledRunTimes(NamespaceId.DEFAULT.getNamespace(), programIds, true);
Assert.assertEquals(programIds.size(), schedules.size());
// For the first 5 programs, they should have a next run
for (int i = 0; i < 5; i++) {
BatchProgramSchedule schedule = schedules.get(i);
Assert.assertEquals(200, schedule.getStatusCode());
List<ScheduledRuntime> nextRuns = schedule.getSchedules();
Assert.assertNotNull(nextRuns);
Assert.assertEquals(1, nextRuns.size());
long nextTime = nextRuns.get(0).getTime();
Assert.assertTrue(nextTime >= now);
}
// The last two should be a not found
Assert.assertEquals(404, schedules.get(5).getStatusCode());
Assert.assertEquals(404, schedules.get(6).getStatusCode());
}
Aggregations