Examples with ScheduleDetail co.cask.cdap.proto.ScheduleDetail used on opensource projects

Example 6 with ScheduleDetail

use of co.cask.cdap.proto.ScheduleDetail in project cdap by caskdata.

the class AuthorizationTest method testScheduleAuth.

@Test
public void testScheduleAuth() throws Exception {
    createAuthNamespace();
    ApplicationId appId = AUTH_NAMESPACE.app(AppWithSchedule.class.getSimpleName());
    Map<EntityId, Set<Action>> neededPrivileges = ImmutableMap.<EntityId, Set<Action>>builder().put(appId, EnumSet.of(Action.ADMIN)).put(AUTH_NAMESPACE.artifact(AppWithSchedule.class.getSimpleName(), "1.0-SNAPSHOT"), EnumSet.of(Action.ADMIN)).put(AUTH_NAMESPACE.dataset(AppWithSchedule.INPUT_NAME), EnumSet.of(Action.ADMIN)).put(AUTH_NAMESPACE.dataset(AppWithSchedule.OUTPUT_NAME), EnumSet.of(Action.ADMIN)).put(AUTH_NAMESPACE.datasetType(ObjectStore.class.getName()), EnumSet.of(Action.ADMIN)).build();
    setUpPrivilegeAndRegisterForDeletion(ALICE, neededPrivileges);
    ApplicationManager appManager = deployApplication(AUTH_NAMESPACE, AppWithSchedule.class);
    String workflowName = AppWithSchedule.SampleWorkflow.class.getSimpleName();
    ProgramId workflowID = new ProgramId(AUTH_NAMESPACE.getNamespace(), AppWithSchedule.class.getSimpleName(), ProgramType.WORKFLOW, workflowName);
    cleanUpEntities.add(workflowID);
    final WorkflowManager workflowManager = appManager.getWorkflowManager(workflowName);
    ScheduleManager scheduleManager = workflowManager.getSchedule(AppWithSchedule.EVERY_HOUR_SCHEDULE);
    // switch to BOB
    SecurityRequestContext.setUserId(BOB.getName());
    // try to resume schedule as BOB. It should fail since BOB does not have execute privileges on the programs
    try {
        scheduleManager.resume();
        Assert.fail("Resuming schedule should have failed since BOB does not have EXECUTE on the program");
    } catch (UnauthorizedException e) {
    // Expected
    }
    // bob should also not be able see the status of the schedule
    try {
        scheduleManager.status(HttpURLConnection.HTTP_FORBIDDEN);
        Assert.fail("Getting schedule status should have failed since BOB does not have any privilege on the program");
    } catch (UnauthorizedException e) {
    // Expected
    }
    // give BOB READ permission in the workflow
    grantAndAssertSuccess(workflowID, BOB, EnumSet.of(Action.READ));
    // switch to BOB
    SecurityRequestContext.setUserId(BOB.getName());
    // try to resume schedule as BOB. It should fail since BOB has READ but not EXECUTE on the workflow
    try {
        scheduleManager.resume();
        Assert.fail("Resuming schedule should have failed since BOB does not have EXECUTE on the program");
    } catch (UnauthorizedException e) {
    // Expected
    }
    // but BOB should be able to get schedule status now
    Assert.assertEquals(ProgramScheduleStatus.SUSPENDED.name(), scheduleManager.status(HttpURLConnection.HTTP_OK));
    // give BOB EXECUTE permission in the workflow
    grantAndAssertSuccess(workflowID, BOB, EnumSet.of(Action.EXECUTE));
    // switch to BOB
    SecurityRequestContext.setUserId(BOB.getName());
    // try to resume the schedule. This should pass and workflow should run
    scheduleManager.resume();
    Assert.assertEquals(ProgramScheduleStatus.SCHEDULED.name(), scheduleManager.status(HttpURLConnection.HTTP_OK));
    // suspend the schedule so that it does not start running again
    scheduleManager.suspend();
    Assert.assertEquals(ProgramScheduleStatus.SUSPENDED.name(), scheduleManager.status(HttpURLConnection.HTTP_OK));
    ScheduleId scheduleId = new ScheduleId(appId.getNamespace(), appId.getApplication(), appId.getVersion(), "testSchedule");
    ScheduleDetail scheduleDetail = new ScheduleDetail(AUTH_NAMESPACE.getNamespace(), AppWithSchedule.class.getSimpleName(), "1.0-SNAPSHOT", "testSchedule", "Something 2", new ScheduleProgramInfo(SchedulableProgramType.WORKFLOW, workflowName), Collections.<String, String>emptyMap(), new TimeTrigger("*/1 * * * *"), Collections.<Constraint>emptyList(), TimeUnit.HOURS.toMillis(6), null);
    try {
        addSchedule(scheduleId, scheduleDetail);
        Assert.fail("Adding schedule should fail since BOB does not have AMDIN on the app");
    } catch (UnauthorizedException e) {
    // expected
    }
    // grant BOB ADMIN on the app
    grantAndAssertSuccess(appId, BOB, EnumSet.of(Action.ADMIN));
    // add schedule should succeed
    addSchedule(scheduleId, scheduleDetail);
    Assert.assertEquals(ProgramScheduleStatus.SUSPENDED.name(), workflowManager.getSchedule(scheduleId.getSchedule()).status(HttpURLConnection.HTTP_OK));
    // update schedule should succeed
    updateSchedule(scheduleId, scheduleDetail);
    Assert.assertEquals(ProgramScheduleStatus.SUSPENDED.name(), workflowManager.getSchedule(scheduleId.getSchedule()).status(HttpURLConnection.HTTP_OK));
    // revoke ADMIN from BOB
    getAuthorizer().revoke(Authorizable.fromEntityId(appId), BOB, EnumSet.of(Action.ADMIN));
    try {
        // delete schedule should fail since we revoke the ADMIN privilege from BOB
        deleteSchedule(scheduleId);
        Assert.fail("Deleting schedule should fail since BOB does not have AMDIN on the app");
    } catch (UnauthorizedException e) {
    // expected
    }
    try {
        updateSchedule(scheduleId, scheduleDetail);
        Assert.fail("Updating schedule should fail since BOB does not have AMDIN on the app");
    } catch (UnauthorizedException e) {
    // expected
    }
    // grant BOB ADMIN on the app again
    grantAndAssertSuccess(appId, BOB, EnumSet.of(Action.ADMIN));
    deleteSchedule(scheduleId);
    workflowManager.getSchedule(scheduleId.getSchedule()).status(HttpURLConnection.HTTP_NOT_FOUND);
    // switch to Alice
    SecurityRequestContext.setUserId(ALICE.getName());
}

Example 7 with ScheduleDetail

use of co.cask.cdap.proto.ScheduleDetail in project cdap by caskdata.

the class TestFrameworkTestRun method testDeployWorkflowApp.

@Category(XSlowTests.class)
@Test
@Ignore
public void testDeployWorkflowApp() throws Exception {
    // Add test back when CDAP-12350 is resolved
    ApplicationManager applicationManager = deployApplication(testSpace, AppWithSchedule.class);
    final WorkflowManager wfmanager = applicationManager.getWorkflowManager(AppWithSchedule.WORKFLOW_NAME);
    List<ScheduleDetail> schedules = wfmanager.getProgramSchedules();
    Assert.assertEquals(2, schedules.size());
    String scheduleName = schedules.get(1).getName();
    Assert.assertNotNull(scheduleName);
    Assert.assertFalse(scheduleName.isEmpty());
    final int initialRuns = wfmanager.getHistory().size();
    LOG.info("initialRuns = {}", initialRuns);
    wfmanager.getSchedule(scheduleName).resume();
    String status = wfmanager.getSchedule(scheduleName).status(200);
    Assert.assertEquals("SCHEDULED", status);
    // Make sure something ran before suspending
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return wfmanager.getHistory().size() > 0;
        }
    }, 15, TimeUnit.SECONDS);
    wfmanager.getSchedule(scheduleName).suspend();
    waitForScheduleState(scheduleName, wfmanager, ProgramScheduleStatus.SUSPENDED);
    // All runs should be completed
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            for (RunRecord record : wfmanager.getHistory()) {
                if (record.getStatus() != ProgramRunStatus.COMPLETED) {
                    return false;
                }
            }
            return true;
        }
    }, 15, TimeUnit.SECONDS);
    List<RunRecord> history = wfmanager.getHistory();
    int workflowRuns = history.size();
    LOG.info("workflowRuns = {}", workflowRuns);
    Assert.assertTrue(workflowRuns > 0);
    // Sleep for some time and verify there are no more scheduled jobs after the suspend.
    TimeUnit.SECONDS.sleep(5);
    final int workflowRunsAfterSuspend = wfmanager.getHistory().size();
    Assert.assertEquals(workflowRuns, workflowRunsAfterSuspend);
    wfmanager.getSchedule(scheduleName).resume();
    // Check that after resume it goes to "SCHEDULED" state
    waitForScheduleState(scheduleName, wfmanager, ProgramScheduleStatus.SCHEDULED);
    // Make sure new runs happens after resume
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return wfmanager.getHistory().size() > workflowRunsAfterSuspend;
        }
    }, 15, TimeUnit.SECONDS);
    // Check scheduled state
    Assert.assertEquals("SCHEDULED", wfmanager.getSchedule(scheduleName).status(200));
    // Check status of non-existent schedule
    Assert.assertEquals("NOT_FOUND", wfmanager.getSchedule("doesnt exist").status(404));
    // Suspend the schedule
    wfmanager.getSchedule(scheduleName).suspend();
    // Check that after suspend it goes to "SUSPENDED" state
    waitForScheduleState(scheduleName, wfmanager, ProgramScheduleStatus.SUSPENDED);
    // Test workflow token while suspended
    String pid = history.get(0).getPid();
    WorkflowTokenDetail workflowToken = wfmanager.getToken(pid, WorkflowToken.Scope.SYSTEM, null);
    Assert.assertEquals(0, workflowToken.getTokenData().size());
    workflowToken = wfmanager.getToken(pid, null, null);
    Assert.assertEquals(2, workflowToken.getTokenData().size());
    // Wait for all workflow runs to finish execution, in case more than one run happened with an enabled schedule
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            for (RunRecord record : wfmanager.getHistory()) {
                if (record.getStatus() != ProgramRunStatus.COMPLETED) {
                    return false;
                }
            }
            return true;
        }
    }, 15, TimeUnit.SECONDS);
    // Verify workflow token after workflow completion
    WorkflowTokenNodeDetail workflowTokenAtNode = wfmanager.getTokenAtNode(pid, AppWithSchedule.DummyAction.class.getSimpleName(), WorkflowToken.Scope.USER, "finished");
    Assert.assertEquals(true, Boolean.parseBoolean(workflowTokenAtNode.getTokenDataAtNode().get("finished")));
    workflowToken = wfmanager.getToken(pid, null, null);
    Assert.assertEquals(false, Boolean.parseBoolean(workflowToken.getTokenData().get("running").get(0).getValue()));
}

Example 8 with ScheduleDetail

use of co.cask.cdap.proto.ScheduleDetail in project cdap by caskdata.

the class DataPipelineTest method deployPipelineWithSchedule.

private WorkflowManager deployPipelineWithSchedule(String pipelineName, Engine engine, String triggeringPipelineName, ArgumentMapping key1Mapping, String expectedKey1Value, PluginPropertyMapping key2Mapping, String expectedKey2Value) throws Exception {
    String tableName = "actionScheduleTable" + pipelineName + engine;
    String sourceName = "macroActionWithScheduleInput-" + pipelineName + engine;
    String sinkName = "macroActionWithScheduleOutput-" + pipelineName + engine;
    String key1 = key1Mapping.getTarget();
    String key2 = key2Mapping.getTarget();
    ETLBatchConfig etlConfig = ETLBatchConfig.builder("* * * * *").addStage(new ETLStage("action1", MockAction.getPlugin(tableName, "row1", "column1", String.format("${%s}", key1)))).addStage(new ETLStage("action2", MockAction.getPlugin(tableName, "row2", "column2", String.format("${%s}", key2)))).addStage(new ETLStage("source", MockSource.getPlugin(sourceName))).addStage(new ETLStage("filter1", StringValueFilterTransform.getPlugin("name", String.format("${%s}", key1)))).addStage(new ETLStage("filter2", StringValueFilterTransform.getPlugin("name", String.format("${%s}", key2)))).addStage(new ETLStage("sink", MockSink.getPlugin(sinkName))).addConnection("action1", "action2").addConnection("action2", "source").addConnection("source", "filter1").addConnection("filter1", "filter2").addConnection("filter2", "sink").setEngine(engine).build();
    AppRequest<ETLBatchConfig> appRequest = new AppRequest<>(APP_ARTIFACT, etlConfig);
    ApplicationId appId = NamespaceId.DEFAULT.app(pipelineName);
    ApplicationManager appManager = deployApplication(appId, appRequest);
    // there should be only two programs - one workflow and one mapreduce/spark
    Schema schema = Schema.recordOf("testRecord", Schema.Field.of("name", Schema.of(Schema.Type.STRING)));
    // Use the expectedKey1Value and expectedKey2Value as values for two records, so that Only record "samuel"
    StructuredRecord recordSamuel = StructuredRecord.builder(schema).set("name", "samuel").build();
    StructuredRecord recordKey1Value = StructuredRecord.builder(schema).set("name", expectedKey1Value).build();
    StructuredRecord recordKey2Value = StructuredRecord.builder(schema).set("name", expectedKey2Value).build();
    // write one record to each source
    DataSetManager<Table> inputManager = getDataset(sourceName);
    MockSource.writeInput(inputManager, ImmutableList.of(recordSamuel, recordKey1Value, recordKey2Value));
    String defaultNamespace = NamespaceId.DEFAULT.getNamespace();
    // Use properties from the triggering pipeline as values for runtime argument key1, key2
    TriggeringPropertyMapping propertyMapping = new TriggeringPropertyMapping(ImmutableList.of(key1Mapping), ImmutableList.of(key2Mapping));
    ProgramStatusTrigger completeTrigger = new ProgramStatusTrigger(new WorkflowId(defaultNamespace, triggeringPipelineName, SmartWorkflow.NAME), ImmutableSet.of(ProgramStatus.COMPLETED));
    ScheduleId scheduleId = appId.schedule("completeSchedule");
    appManager.addSchedule(new ScheduleDetail(scheduleId.getNamespace(), scheduleId.getApplication(), scheduleId.getVersion(), scheduleId.getSchedule(), "", new ScheduleProgramInfo(SchedulableProgramType.WORKFLOW, SmartWorkflow.NAME), ImmutableMap.of(SmartWorkflow.TRIGGERING_PROPERTIES_MAPPING, GSON.toJson(propertyMapping)), completeTrigger, ImmutableList.<Constraint>of(), Schedulers.JOB_QUEUE_TIMEOUT_MILLIS, null));
    appManager.enableSchedule(scheduleId);
    return appManager.getWorkflowManager(SmartWorkflow.NAME);
}

Example 9 with ScheduleDetail

use of co.cask.cdap.proto.ScheduleDetail in project cdap by caskdata.

the class AppScheduleUpdateTest method testUpdateSchedulesFlag.

@Test
public void testUpdateSchedulesFlag() throws Exception {
    // deploy an app with schedule
    AppWithSchedule.AppConfig config = new AppWithSchedule.AppConfig(true, true, true);
    Id.Artifact artifactId = Id.Artifact.from(Id.Namespace.fromEntityId(TEST_NAMESPACE_META2.getNamespaceId()), AppWithSchedule.NAME, VERSION1);
    addAppArtifact(artifactId, AppWithSchedule.class);
    AppRequest<? extends Config> request = new AppRequest<>(new ArtifactSummary(artifactId.getName(), artifactId.getVersion().getVersion()), config);
    ApplicationId defaultAppId = TEST_NAMESPACE_META2.getNamespaceId().app(AppWithSchedule.NAME);
    Assert.assertEquals(200, deploy(defaultAppId, request).getStatusLine().getStatusCode());
    List<ScheduleDetail> actualSchSpecs = listSchedules(TEST_NAMESPACE_META2.getNamespaceId().getNamespace(), defaultAppId.getApplication(), defaultAppId.getVersion());
    // none of the schedules will be added - by default we have set update schedules to be false as system property.
    Assert.assertEquals(0, actualSchSpecs.size());
    request = new AppRequest<>(new ArtifactSummary(artifactId.getName(), artifactId.getVersion().getVersion()), config, null, null, true);
    Assert.assertEquals(200, deploy(defaultAppId, request).getStatusLine().getStatusCode());
    actualSchSpecs = listSchedules(TEST_NAMESPACE_META2.getNamespaceId().getNamespace(), defaultAppId.getApplication(), defaultAppId.getVersion());
    // both the schedules will be added as now,
    // we have provided update schedules property to be true manually in appRequest
    Assert.assertEquals(2, actualSchSpecs.size());
    config = new AppWithSchedule.AppConfig(true, true, false);
    request = new AppRequest<>(new ArtifactSummary(artifactId.getName(), artifactId.getVersion().getVersion()), config);
    Assert.assertEquals(200, deploy(defaultAppId, request).getStatusLine().getStatusCode());
    actualSchSpecs = listSchedules(TEST_NAMESPACE_META2.getNamespaceId().getNamespace(), defaultAppId.getApplication(), defaultAppId.getVersion());
    // no changes will be made, as default behavior is dont update schedules, so both the schedules should be there
    Assert.assertEquals(2, actualSchSpecs.size());
    config = new AppWithSchedule.AppConfig(false, false, false);
    request = new AppRequest<>(new ArtifactSummary(artifactId.getName(), artifactId.getVersion().getVersion()), config);
    Assert.assertEquals(200, deploy(defaultAppId, request).getStatusLine().getStatusCode());
    actualSchSpecs = listSchedules(TEST_NAMESPACE_META2.getNamespaceId().getNamespace(), defaultAppId.getApplication(), defaultAppId.getVersion());
    // workflow is deleted, so the schedules will be deleted now
    Assert.assertEquals(0, actualSchSpecs.size());
}

Example 10 with ScheduleDetail

use of co.cask.cdap.proto.ScheduleDetail in project cdap by caskdata.

the class ProgramLifecycleHttpHandlerTest method testUpdateSchedule.

private void testUpdateSchedule(ApplicationId appV2Id) throws Exception {
    ScheduleDetail updateDetail = new ScheduleDetail(AppWithSchedule.SCHEDULE, "updatedDescription", null, ImmutableMap.of("twoKey", "twoValue", "someKey", "newValue"), new TimeTrigger("0 4 * * *"), ImmutableList.of(new ConcurrencyConstraint(5)), null);
    // trying to update schedule for a non-existing app should fail
    HttpResponse response = updateSchedule(TEST_NAMESPACE1, "nonExistingApp", null, AppWithSchedule.SCHEDULE, updateDetail);
    Assert.assertEquals(HttpResponseStatus.NOT_FOUND.code(), response.getStatusLine().getStatusCode());
    // trying to update a non-existing schedule should fail
    ScheduleDetail nonExistingSchedule = new ScheduleDetail("NonExistingSchedule", "updatedDescription", null, ImmutableMap.of("twoKey", "twoValue"), new TimeTrigger("0 4 * * *"), ImmutableList.of(new ConcurrencyConstraint(5)), null);
    response = updateSchedule(TEST_NAMESPACE1, AppWithSchedule.NAME, null, "NonExistingSchedule", nonExistingSchedule);
    Assert.assertEquals(HttpResponseStatus.NOT_FOUND.code(), response.getStatusLine().getStatusCode());
    // should be able to update an existing schedule with a valid new time schedule
    response = updateSchedule(TEST_NAMESPACE1, AppWithSchedule.NAME, null, AppWithSchedule.SCHEDULE, updateDetail);
    Assert.assertEquals(HttpResponseStatus.OK.code(), response.getStatusLine().getStatusCode());
    // verify that the schedule information for updated
    ScheduleDetail schedule = getSchedule(TEST_NAMESPACE1, AppWithSchedule.NAME, null, AppWithSchedule.SCHEDULE);
    Assert.assertEquals("updatedDescription", schedule.getDescription());
    Assert.assertEquals("0 4 * * *", ((TimeTrigger) schedule.getTrigger()).getCronExpression());
    Assert.assertEquals(new ProtoConstraint.ConcurrencyConstraint(5), schedule.getConstraints().get(0));
    // the properties should have been replaced
    Assert.assertEquals(2, schedule.getProperties().size());
    Assert.assertEquals("newValue", schedule.getProperties().get("someKey"));
    Assert.assertEquals("twoValue", schedule.getProperties().get("twoKey"));
    // the old property should not exist
    Assert.assertNull(schedule.getProperties().get("oneKey"));
    // the above update should not have affected the schedule for the other version of the app
    schedule = getSchedule(TEST_NAMESPACE1, AppWithSchedule.NAME, appV2Id.getVersion(), AppWithSchedule.SCHEDULE);
    Assert.assertNotEquals("updatedDescription", schedule.getDescription());
    Assert.assertEquals("0/15 * * * * ?", ((TimeTrigger) schedule.getTrigger()).getCronExpression());
    // try to update the schedule again but this time with property as null. It should retain the old properties
    ScheduleDetail scheduleDetail = new ScheduleDetail(AppWithSchedule.SCHEDULE, "updatedDescription", null, null, new ProtoTrigger.TimeTrigger("0 4 * * *"), null, null);
    response = updateSchedule(TEST_NAMESPACE1, AppWithSchedule.NAME, null, AppWithSchedule.SCHEDULE, scheduleDetail);
    Assert.assertEquals(HttpResponseStatus.OK.code(), response.getStatusLine().getStatusCode());
    schedule = getSchedule(TEST_NAMESPACE1, AppWithSchedule.NAME, null, AppWithSchedule.SCHEDULE);
    Assert.assertEquals(2, schedule.getProperties().size());
    Assert.assertEquals("newValue", schedule.getProperties().get("someKey"));
    Assert.assertEquals("twoValue", schedule.getProperties().get("twoKey"));
    Assert.assertEquals(new ProtoConstraint.ConcurrencyConstraint(5), schedule.getConstraints().get(0));
}