use of io.cdap.cdap.app.store.Store in project cdap by caskdata.
the class ProfileServiceTest method testProfileDeletion.
@Test
public void testProfileDeletion() throws Exception {
ProfileId myProfile = NamespaceId.DEFAULT.profile("MyProfile");
ProfileId myProfile2 = NamespaceId.DEFAULT.profile("MyProfile2");
Profile profile1 = new Profile("MyProfile", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), Profile.NATIVE.getProvisioner());
Profile profile2 = new Profile("MyProfile2", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), ProfileStatus.DISABLED, Profile.NATIVE.getProvisioner());
profileService.saveProfile(myProfile, profile1);
// add profile2 and disable it, profile2 can get deleted at any time
profileService.saveProfile(myProfile2, profile2);
profileService.disableProfile(myProfile2);
// Should not be able to delete because the profile is by default enabled
try {
profileService.deleteProfile(myProfile);
Assert.fail();
} catch (ProfileConflictException e) {
// expected
}
try {
profileService.deleteAllProfiles(NamespaceId.DEFAULT);
Assert.fail();
} catch (ProfileConflictException e) {
// expected and check profile 2 is not getting deleted
Assert.assertEquals(profile2, profileService.getProfile(myProfile2));
}
// add assignment and disable it, deletion should also fail
profileService.addProfileAssignment(myProfile, NamespaceId.DEFAULT);
profileService.disableProfile(myProfile);
try {
profileService.deleteProfile(myProfile);
Assert.fail();
} catch (ProfileConflictException e) {
// expected
}
try {
profileService.deleteAllProfiles(NamespaceId.DEFAULT);
Assert.fail();
} catch (ProfileConflictException e) {
// expected and check profile 2 is not getting deleted
Assert.assertEquals(profile2, profileService.getProfile(myProfile2));
}
profileService.removeProfileAssignment(myProfile, NamespaceId.DEFAULT);
// add an active record to DefaultStore, deletion should still fail
Store store = getDefaultStore();
ProgramId programId = NamespaceId.DEFAULT.app("myApp").workflow("myProgram");
ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
RunId runId = RunIds.generate(System.currentTimeMillis());
ProgramRunId programRunId = programId.run(runId.getId());
Map<String, String> systemArgs = Collections.singletonMap(SystemArguments.PROFILE_NAME, myProfile.getScopedName());
int sourceId = 0;
store.setProvisioning(programRunId, Collections.emptyMap(), systemArgs, AppFabricTestHelper.createSourceId(++sourceId), artifactId);
store.setProvisioned(programRunId, 0, AppFabricTestHelper.createSourceId(++sourceId));
store.setStart(programRunId, null, systemArgs, AppFabricTestHelper.createSourceId(++sourceId));
try {
profileService.deleteProfile(myProfile);
Assert.fail();
} catch (ProfileConflictException e) {
// expected
}
try {
profileService.deleteAllProfiles(NamespaceId.DEFAULT);
Assert.fail();
} catch (ProfileConflictException e) {
// expected and check profile 2 is not getting deleted
Assert.assertEquals(profile2, profileService.getProfile(myProfile2));
}
// set the run to stopped then deletion should work
store.setStop(programRunId, System.currentTimeMillis() + 1000, ProgramController.State.ERROR.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId));
// now profile deletion should succeed
profileService.deleteProfile(myProfile);
Assert.assertEquals(Collections.singletonList(profile2), profileService.getProfiles(NamespaceId.DEFAULT, false));
profileService.saveProfile(myProfile, profile1);
profileService.disableProfile(myProfile);
profileService.deleteAllProfiles(NamespaceId.DEFAULT);
Assert.assertEquals(Collections.emptyList(), profileService.getProfiles(NamespaceId.DEFAULT, false));
}
use of io.cdap.cdap.app.store.Store in project cdap by caskdata.
the class MetadataSubscriberServiceTest method testProfileMetadata.
@Test
public void testProfileMetadata() throws Exception {
Injector injector = getInjector();
ApplicationSpecification appSpec = Specifications.from(new AppWithWorkflow());
ApplicationId appId = NamespaceId.DEFAULT.app(appSpec.getName());
ProgramId workflowId = appId.workflow("SampleWorkflow");
ScheduleId scheduleId = appId.schedule("tsched1");
// publish a creation of a schedule that will never exist
// this tests that such a message is eventually discarded
// note that for this test, we configure a fast retry strategy and a small number of retries
// therefore this will cost only a few seconds delay
publishBogusCreationEvent();
// get the mds should be empty property since we haven't started the MetadataSubscriberService
MetadataStorage mds = injector.getInstance(MetadataStorage.class);
Assert.assertEquals(Collections.emptyMap(), mds.read(new Read(workflowId.toMetadataEntity())).getProperties());
Assert.assertEquals(Collections.emptyMap(), mds.read(new Read(scheduleId.toMetadataEntity())).getProperties());
// add a app with workflow to app meta store
// note: since we bypass the app-fabric when adding this app, no ENTITY_CREATION message
// will be published for the app (it happens in app lifecycle service). Therefore this
// app must exist before assigning the profile for the namespace, otherwise the app's
// programs will not receive the profile metadata.
Store store = injector.getInstance(DefaultStore.class);
store.addApplication(appId, appSpec);
// set default namespace to use the profile, since now MetadataSubscriberService is not started,
// it should not affect the mds
PreferencesService preferencesService = injector.getInstance(PreferencesService.class);
preferencesService.setProperties(NamespaceId.DEFAULT, Collections.singletonMap(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()));
// add a schedule to schedule store
ProgramScheduleService scheduleService = injector.getInstance(ProgramScheduleService.class);
scheduleService.add(new ProgramSchedule("tsched1", "one time schedule", workflowId, Collections.emptyMap(), new TimeTrigger("* * ? * 1"), ImmutableList.of()));
// add a new profile in default namespace
ProfileService profileService = injector.getInstance(ProfileService.class);
ProfileId myProfile = new ProfileId(NamespaceId.DEFAULT.getNamespace(), "MyProfile");
Profile profile1 = new Profile("MyProfile", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), Profile.NATIVE.getProvisioner());
profileService.saveProfile(myProfile, profile1);
// add a second profile in default namespace
ProfileId myProfile2 = new ProfileId(NamespaceId.DEFAULT.getNamespace(), "MyProfile2");
Profile profile2 = new Profile("MyProfile2", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), Profile.NATIVE.getProvisioner());
profileService.saveProfile(myProfile2, profile2);
try {
// Verify the workflow profile metadata is updated to default profile
Tasks.waitFor(ProfileId.NATIVE.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Verify the schedule profile metadata is updated to default profile
Tasks.waitFor(ProfileId.NATIVE.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// set default namespace to use my profile
preferencesService.setProperties(NamespaceId.DEFAULT, Collections.singletonMap(SystemArguments.PROFILE_NAME, "USER:MyProfile"));
// Verify the workflow profile metadata is updated to my profile
Tasks.waitFor(myProfile.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Verify the schedule profile metadata is updated to my profile
Tasks.waitFor(myProfile.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// set app level to use my profile 2
preferencesService.setProperties(appId, Collections.singletonMap(SystemArguments.PROFILE_NAME, "USER:MyProfile2"));
// set instance level to system profile
preferencesService.setProperties(Collections.singletonMap(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()));
// Verify the workflow profile metadata is updated to MyProfile2 which is at app level
Tasks.waitFor(myProfile2.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Verify the schedule profile metadata is updated to MyProfile2 which is at app level
Tasks.waitFor(myProfile2.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// remove the preferences at instance level, should not affect the metadata
preferencesService.deleteProperties();
// Verify the workflow profile metadata is updated to default profile
Tasks.waitFor(myProfile2.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Verify the schedule profile metadata is updated to default profile
Tasks.waitFor(myProfile2.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// remove app level pref should let the programs/schedules use ns level pref
preferencesService.deleteProperties(appId);
// Verify the workflow profile metadata is updated to MyProfile
Tasks.waitFor(myProfile.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Verify the schedule profile metadata is updated to MyProfile
Tasks.waitFor(myProfile.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// remove ns level pref so no pref is there
preferencesService.deleteProperties(NamespaceId.DEFAULT);
// Verify the workflow profile metadata is updated to default profile
Tasks.waitFor(ProfileId.NATIVE.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Verify the schedule profile metadata is updated to default profile
Tasks.waitFor(ProfileId.NATIVE.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
} finally {
// stop and clean up the store
preferencesService.deleteProperties(NamespaceId.DEFAULT);
preferencesService.deleteProperties();
preferencesService.deleteProperties(appId);
store.removeAll(NamespaceId.DEFAULT);
scheduleService.delete(scheduleId);
profileService.disableProfile(myProfile);
profileService.disableProfile(myProfile2);
profileService.deleteAllProfiles(myProfile.getNamespaceId());
mds.apply(new MetadataMutation.Drop(workflowId.toMetadataEntity()), MutationOptions.DEFAULT);
mds.apply(new MetadataMutation.Drop(scheduleId.toMetadataEntity()), MutationOptions.DEFAULT);
}
}
use of io.cdap.cdap.app.store.Store in project cdap by caskdata.
the class MetadataSubscriberServiceTest method testWorkflow.
@Test
public void testWorkflow() throws InterruptedException, ExecutionException, TimeoutException {
ProgramRunId workflowRunId = workflow1.run(RunIds.generate());
// Try to read, should have nothing
Store store = getInjector().getInstance(DefaultStore.class);
WorkflowToken workflowToken = store.getWorkflowToken(workflow1, workflowRunId.getRun());
Assert.assertNull(workflowToken.get("key"));
BasicWorkflowToken token = new BasicWorkflowToken(1024);
token.setCurrentNode("node1");
token.put("key", "value");
// Publish some workflow states
WorkflowStateWriter workflowStateWriter = getInjector().getInstance(MessagingWorkflowStateWriter.class);
workflowStateWriter.setWorkflowToken(workflowRunId, token);
workflowStateWriter.addWorkflowNodeState(workflowRunId, new WorkflowNodeStateDetail("action1", NodeStatus.RUNNING));
// Verify the WorkflowToken
Tasks.waitFor("value", () -> Optional.ofNullable(store.getWorkflowToken(workflow1, workflowRunId.getRun()).get("key")).map(Value::toString).orElse(null), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Verify the workflow node state
Tasks.waitFor(NodeStatus.RUNNING, () -> store.getWorkflowNodeStates(workflowRunId).stream().findFirst().map(WorkflowNodeStateDetail::getNodeStatus).orElse(null), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Update the node state
workflowStateWriter.addWorkflowNodeState(workflowRunId, new WorkflowNodeStateDetail("action1", NodeStatus.COMPLETED));
// Verify the updated node state
Tasks.waitFor(NodeStatus.COMPLETED, () -> store.getWorkflowNodeStates(workflowRunId).stream().findFirst().map(WorkflowNodeStateDetail::getNodeStatus).orElse(null), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
}
use of io.cdap.cdap.app.store.Store in project cdap by cdapio.
the class ProfileServiceTest method testProfileDeletion.
@Test
public void testProfileDeletion() throws Exception {
ProfileId myProfile = NamespaceId.DEFAULT.profile("MyProfile");
ProfileId myProfile2 = NamespaceId.DEFAULT.profile("MyProfile2");
Profile profile1 = new Profile("MyProfile", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), Profile.NATIVE.getProvisioner());
Profile profile2 = new Profile("MyProfile2", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), ProfileStatus.DISABLED, Profile.NATIVE.getProvisioner());
profileService.saveProfile(myProfile, profile1);
// add profile2 and disable it, profile2 can get deleted at any time
profileService.saveProfile(myProfile2, profile2);
profileService.disableProfile(myProfile2);
// Should not be able to delete because the profile is by default enabled
try {
profileService.deleteProfile(myProfile);
Assert.fail();
} catch (ProfileConflictException e) {
// expected
}
try {
profileService.deleteAllProfiles(NamespaceId.DEFAULT);
Assert.fail();
} catch (ProfileConflictException e) {
// expected and check profile 2 is not getting deleted
Assert.assertEquals(profile2, profileService.getProfile(myProfile2));
}
// add assignment and disable it, deletion should also fail
profileService.addProfileAssignment(myProfile, NamespaceId.DEFAULT);
profileService.disableProfile(myProfile);
try {
profileService.deleteProfile(myProfile);
Assert.fail();
} catch (ProfileConflictException e) {
// expected
}
try {
profileService.deleteAllProfiles(NamespaceId.DEFAULT);
Assert.fail();
} catch (ProfileConflictException e) {
// expected and check profile 2 is not getting deleted
Assert.assertEquals(profile2, profileService.getProfile(myProfile2));
}
profileService.removeProfileAssignment(myProfile, NamespaceId.DEFAULT);
// add an active record to DefaultStore, deletion should still fail
Store store = getDefaultStore();
ProgramId programId = NamespaceId.DEFAULT.app("myApp").workflow("myProgram");
ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
RunId runId = RunIds.generate(System.currentTimeMillis());
ProgramRunId programRunId = programId.run(runId.getId());
Map<String, String> systemArgs = Collections.singletonMap(SystemArguments.PROFILE_NAME, myProfile.getScopedName());
int sourceId = 0;
store.setProvisioning(programRunId, Collections.emptyMap(), systemArgs, AppFabricTestHelper.createSourceId(++sourceId), artifactId);
store.setProvisioned(programRunId, 0, AppFabricTestHelper.createSourceId(++sourceId));
store.setStart(programRunId, null, systemArgs, AppFabricTestHelper.createSourceId(++sourceId));
try {
profileService.deleteProfile(myProfile);
Assert.fail();
} catch (ProfileConflictException e) {
// expected
}
try {
profileService.deleteAllProfiles(NamespaceId.DEFAULT);
Assert.fail();
} catch (ProfileConflictException e) {
// expected and check profile 2 is not getting deleted
Assert.assertEquals(profile2, profileService.getProfile(myProfile2));
}
// set the run to stopped then deletion should work
store.setStop(programRunId, System.currentTimeMillis() + 1000, ProgramController.State.ERROR.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId));
// now profile deletion should succeed
profileService.deleteProfile(myProfile);
Assert.assertEquals(Collections.singletonList(profile2), profileService.getProfiles(NamespaceId.DEFAULT, false));
profileService.saveProfile(myProfile, profile1);
profileService.disableProfile(myProfile);
profileService.deleteAllProfiles(NamespaceId.DEFAULT);
Assert.assertEquals(Collections.emptyList(), profileService.getProfiles(NamespaceId.DEFAULT, false));
}
use of io.cdap.cdap.app.store.Store in project cdap by cdapio.
the class LineageAdminTest method testSimpleLoopLineage.
@Test
public void testSimpleLoopLineage() {
// Lineage for D1 -> P1 -> D2 -> P2 -> D3 -> P3 -> D4
// | |
// | V
// |<-----------------
//
TransactionRunner transactionRunner = getInjector().getInstance(TransactionRunner.class);
LineageStoreReader lineageReader = new DefaultLineageStoreReader(transactionRunner);
LineageWriter lineageWriter = new BasicLineageWriter(transactionRunner);
Store store = getInjector().getInstance(Store.class);
LineageAdmin lineageAdmin = new LineageAdmin(lineageReader, store);
// Add access
addRuns(store, run1, run2, run3, run4, run5);
// It is okay to use current time here since access time is ignore during assertions
lineageWriter.addAccess(run1, dataset1, AccessType.READ);
lineageWriter.addAccess(run1, dataset2, AccessType.WRITE);
lineageWriter.addAccess(run2, dataset2, AccessType.READ);
lineageWriter.addAccess(run2, dataset1, AccessType.WRITE);
lineageWriter.addAccess(run2, dataset3, AccessType.WRITE);
lineageWriter.addAccess(run3, dataset3, AccessType.READ, null);
lineageWriter.addAccess(run3, dataset4, AccessType.WRITE, null);
Lineage expectedLineage = new Lineage(ImmutableSet.of(new Relation(dataset2, program1, AccessType.WRITE, twillRunId(run1)), new Relation(dataset1, program1, AccessType.READ, twillRunId(run1)), new Relation(dataset1, program2, AccessType.WRITE, twillRunId(run2)), new Relation(dataset2, program2, AccessType.READ, twillRunId(run2)), new Relation(dataset3, program2, AccessType.WRITE, twillRunId(run2)), new Relation(dataset4, program3, AccessType.WRITE, twillRunId(run3)), new Relation(dataset3, program3, AccessType.READ, twillRunId(run3))));
// Lineage for D1
Assert.assertEquals(expectedLineage, lineageAdmin.computeLineage(dataset1, 500, 20000, 100));
// Lineage for D2
Assert.assertEquals(expectedLineage, lineageAdmin.computeLineage(dataset2, 500, 20000, 100));
// Lineage for D1 for one level D1 -> P1 -> D2 -> P2 -> D3
// | |
// | V
// |<-----------------
//
Lineage oneLevelLineage = lineageAdmin.computeLineage(dataset1, 500, 20000, 1);
Assert.assertEquals(ImmutableSet.of(new Relation(dataset2, program1, AccessType.WRITE, twillRunId(run1)), new Relation(dataset1, program1, AccessType.READ, twillRunId(run1)), new Relation(dataset1, program2, AccessType.WRITE, twillRunId(run2)), new Relation(dataset2, program2, AccessType.READ, twillRunId(run2)), new Relation(dataset3, program2, AccessType.WRITE, twillRunId(run2))), oneLevelLineage.getRelations());
}
Aggregations