Examples with Store io.cdap.cdap.app.store.Store used on opensource projects

Example 6 with Store

use of io.cdap.cdap.app.store.Store in project cdap by caskdata.

the class ProfileServiceTest method testProfileDeletion.

@Test
public void testProfileDeletion() throws Exception {
    ProfileId myProfile = NamespaceId.DEFAULT.profile("MyProfile");
    ProfileId myProfile2 = NamespaceId.DEFAULT.profile("MyProfile2");
    Profile profile1 = new Profile("MyProfile", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), Profile.NATIVE.getProvisioner());
    Profile profile2 = new Profile("MyProfile2", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), ProfileStatus.DISABLED, Profile.NATIVE.getProvisioner());
    profileService.saveProfile(myProfile, profile1);
    // add profile2 and disable it, profile2 can get deleted at any time
    profileService.saveProfile(myProfile2, profile2);
    profileService.disableProfile(myProfile2);
    // Should not be able to delete because the profile is by default enabled
    try {
        profileService.deleteProfile(myProfile);
        Assert.fail();
    } catch (ProfileConflictException e) {
    // expected
    }
    try {
        profileService.deleteAllProfiles(NamespaceId.DEFAULT);
        Assert.fail();
    } catch (ProfileConflictException e) {
        // expected and check profile 2 is not getting deleted
        Assert.assertEquals(profile2, profileService.getProfile(myProfile2));
    }
    // add assignment and disable it, deletion should also fail
    profileService.addProfileAssignment(myProfile, NamespaceId.DEFAULT);
    profileService.disableProfile(myProfile);
    try {
        profileService.deleteProfile(myProfile);
        Assert.fail();
    } catch (ProfileConflictException e) {
    // expected
    }
    try {
        profileService.deleteAllProfiles(NamespaceId.DEFAULT);
        Assert.fail();
    } catch (ProfileConflictException e) {
        // expected and check profile 2 is not getting deleted
        Assert.assertEquals(profile2, profileService.getProfile(myProfile2));
    }
    profileService.removeProfileAssignment(myProfile, NamespaceId.DEFAULT);
    // add an active record to DefaultStore, deletion should still fail
    Store store = getDefaultStore();
    ProgramId programId = NamespaceId.DEFAULT.app("myApp").workflow("myProgram");
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
    RunId runId = RunIds.generate(System.currentTimeMillis());
    ProgramRunId programRunId = programId.run(runId.getId());
    Map<String, String> systemArgs = Collections.singletonMap(SystemArguments.PROFILE_NAME, myProfile.getScopedName());
    int sourceId = 0;
    store.setProvisioning(programRunId, Collections.emptyMap(), systemArgs, AppFabricTestHelper.createSourceId(++sourceId), artifactId);
    store.setProvisioned(programRunId, 0, AppFabricTestHelper.createSourceId(++sourceId));
    store.setStart(programRunId, null, systemArgs, AppFabricTestHelper.createSourceId(++sourceId));
    try {
        profileService.deleteProfile(myProfile);
        Assert.fail();
    } catch (ProfileConflictException e) {
    // expected
    }
    try {
        profileService.deleteAllProfiles(NamespaceId.DEFAULT);
        Assert.fail();
    } catch (ProfileConflictException e) {
        // expected and check profile 2 is not getting deleted
        Assert.assertEquals(profile2, profileService.getProfile(myProfile2));
    }
    // set the run to stopped then deletion should work
    store.setStop(programRunId, System.currentTimeMillis() + 1000, ProgramController.State.ERROR.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId));
    // now profile deletion should succeed
    profileService.deleteProfile(myProfile);
    Assert.assertEquals(Collections.singletonList(profile2), profileService.getProfiles(NamespaceId.DEFAULT, false));
    profileService.saveProfile(myProfile, profile1);
    profileService.disableProfile(myProfile);
    profileService.deleteAllProfiles(NamespaceId.DEFAULT);
    Assert.assertEquals(Collections.emptyList(), profileService.getProfiles(NamespaceId.DEFAULT, false));
}

Example 7 with Store

use of io.cdap.cdap.app.store.Store in project cdap by caskdata.

the class MetadataSubscriberServiceTest method testProfileMetadata.

@Test
public void testProfileMetadata() throws Exception {
    Injector injector = getInjector();
    ApplicationSpecification appSpec = Specifications.from(new AppWithWorkflow());
    ApplicationId appId = NamespaceId.DEFAULT.app(appSpec.getName());
    ProgramId workflowId = appId.workflow("SampleWorkflow");
    ScheduleId scheduleId = appId.schedule("tsched1");
    // publish a creation of a schedule that will never exist
    // this tests that such a message is eventually discarded
    // note that for this test, we configure a fast retry strategy and a small number of retries
    // therefore this will cost only a few seconds delay
    publishBogusCreationEvent();
    // get the mds should be empty property since we haven't started the MetadataSubscriberService
    MetadataStorage mds = injector.getInstance(MetadataStorage.class);
    Assert.assertEquals(Collections.emptyMap(), mds.read(new Read(workflowId.toMetadataEntity())).getProperties());
    Assert.assertEquals(Collections.emptyMap(), mds.read(new Read(scheduleId.toMetadataEntity())).getProperties());
    // add a app with workflow to app meta store
    // note: since we bypass the app-fabric when adding this app, no ENTITY_CREATION message
    // will be published for the app (it happens in app lifecycle service). Therefore this
    // app must exist before assigning the profile for the namespace, otherwise the app's
    // programs will not receive the profile metadata.
    Store store = injector.getInstance(DefaultStore.class);
    store.addApplication(appId, appSpec);
    // set default namespace to use the profile, since now MetadataSubscriberService is not started,
    // it should not affect the mds
    PreferencesService preferencesService = injector.getInstance(PreferencesService.class);
    preferencesService.setProperties(NamespaceId.DEFAULT, Collections.singletonMap(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()));
    // add a schedule to schedule store
    ProgramScheduleService scheduleService = injector.getInstance(ProgramScheduleService.class);
    scheduleService.add(new ProgramSchedule("tsched1", "one time schedule", workflowId, Collections.emptyMap(), new TimeTrigger("* * ? * 1"), ImmutableList.of()));
    // add a new profile in default namespace
    ProfileService profileService = injector.getInstance(ProfileService.class);
    ProfileId myProfile = new ProfileId(NamespaceId.DEFAULT.getNamespace(), "MyProfile");
    Profile profile1 = new Profile("MyProfile", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), Profile.NATIVE.getProvisioner());
    profileService.saveProfile(myProfile, profile1);
    // add a second profile in default namespace
    ProfileId myProfile2 = new ProfileId(NamespaceId.DEFAULT.getNamespace(), "MyProfile2");
    Profile profile2 = new Profile("MyProfile2", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), Profile.NATIVE.getProvisioner());
    profileService.saveProfile(myProfile2, profile2);
    try {
        // Verify the workflow profile metadata is updated to default profile
        Tasks.waitFor(ProfileId.NATIVE.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // Verify the schedule profile metadata is updated to default profile
        Tasks.waitFor(ProfileId.NATIVE.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // set default namespace to use my profile
        preferencesService.setProperties(NamespaceId.DEFAULT, Collections.singletonMap(SystemArguments.PROFILE_NAME, "USER:MyProfile"));
        // Verify the workflow profile metadata is updated to my profile
        Tasks.waitFor(myProfile.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // Verify the schedule profile metadata is updated to my profile
        Tasks.waitFor(myProfile.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // set app level to use my profile 2
        preferencesService.setProperties(appId, Collections.singletonMap(SystemArguments.PROFILE_NAME, "USER:MyProfile2"));
        // set instance level to system profile
        preferencesService.setProperties(Collections.singletonMap(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()));
        // Verify the workflow profile metadata is updated to MyProfile2 which is at app level
        Tasks.waitFor(myProfile2.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // Verify the schedule profile metadata is updated to MyProfile2 which is at app level
        Tasks.waitFor(myProfile2.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // remove the preferences at instance level, should not affect the metadata
        preferencesService.deleteProperties();
        // Verify the workflow profile metadata is updated to default profile
        Tasks.waitFor(myProfile2.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // Verify the schedule profile metadata is updated to default profile
        Tasks.waitFor(myProfile2.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // remove app level pref should let the programs/schedules use ns level pref
        preferencesService.deleteProperties(appId);
        // Verify the workflow profile metadata is updated to MyProfile
        Tasks.waitFor(myProfile.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // Verify the schedule profile metadata is updated to MyProfile
        Tasks.waitFor(myProfile.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // remove ns level pref so no pref is there
        preferencesService.deleteProperties(NamespaceId.DEFAULT);
        // Verify the workflow profile metadata is updated to default profile
        Tasks.waitFor(ProfileId.NATIVE.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // Verify the schedule profile metadata is updated to default profile
        Tasks.waitFor(ProfileId.NATIVE.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    } finally {
        // stop and clean up the store
        preferencesService.deleteProperties(NamespaceId.DEFAULT);
        preferencesService.deleteProperties();
        preferencesService.deleteProperties(appId);
        store.removeAll(NamespaceId.DEFAULT);
        scheduleService.delete(scheduleId);
        profileService.disableProfile(myProfile);
        profileService.disableProfile(myProfile2);
        profileService.deleteAllProfiles(myProfile.getNamespaceId());
        mds.apply(new MetadataMutation.Drop(workflowId.toMetadataEntity()), MutationOptions.DEFAULT);
        mds.apply(new MetadataMutation.Drop(scheduleId.toMetadataEntity()), MutationOptions.DEFAULT);
    }
}

Example 8 with Store

use of io.cdap.cdap.app.store.Store in project cdap by caskdata.

the class MetadataSubscriberServiceTest method testWorkflow.

@Test
public void testWorkflow() throws InterruptedException, ExecutionException, TimeoutException {
    ProgramRunId workflowRunId = workflow1.run(RunIds.generate());
    // Try to read, should have nothing
    Store store = getInjector().getInstance(DefaultStore.class);
    WorkflowToken workflowToken = store.getWorkflowToken(workflow1, workflowRunId.getRun());
    Assert.assertNull(workflowToken.get("key"));
    BasicWorkflowToken token = new BasicWorkflowToken(1024);
    token.setCurrentNode("node1");
    token.put("key", "value");
    // Publish some workflow states
    WorkflowStateWriter workflowStateWriter = getInjector().getInstance(MessagingWorkflowStateWriter.class);
    workflowStateWriter.setWorkflowToken(workflowRunId, token);
    workflowStateWriter.addWorkflowNodeState(workflowRunId, new WorkflowNodeStateDetail("action1", NodeStatus.RUNNING));
    // Verify the WorkflowToken
    Tasks.waitFor("value", () -> Optional.ofNullable(store.getWorkflowToken(workflow1, workflowRunId.getRun()).get("key")).map(Value::toString).orElse(null), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Verify the workflow node state
    Tasks.waitFor(NodeStatus.RUNNING, () -> store.getWorkflowNodeStates(workflowRunId).stream().findFirst().map(WorkflowNodeStateDetail::getNodeStatus).orElse(null), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Update the node state
    workflowStateWriter.addWorkflowNodeState(workflowRunId, new WorkflowNodeStateDetail("action1", NodeStatus.COMPLETED));
    // Verify the updated node state
    Tasks.waitFor(NodeStatus.COMPLETED, () -> store.getWorkflowNodeStates(workflowRunId).stream().findFirst().map(WorkflowNodeStateDetail::getNodeStatus).orElse(null), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
}

Example 9 with Store

use of io.cdap.cdap.app.store.Store in project cdap by cdapio.

the class ProfileServiceTest method testProfileDeletion.

@Test
public void testProfileDeletion() throws Exception {
    ProfileId myProfile = NamespaceId.DEFAULT.profile("MyProfile");
    ProfileId myProfile2 = NamespaceId.DEFAULT.profile("MyProfile2");
    Profile profile1 = new Profile("MyProfile", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), Profile.NATIVE.getProvisioner());
    Profile profile2 = new Profile("MyProfile2", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), ProfileStatus.DISABLED, Profile.NATIVE.getProvisioner());
    profileService.saveProfile(myProfile, profile1);
    // add profile2 and disable it, profile2 can get deleted at any time
    profileService.saveProfile(myProfile2, profile2);
    profileService.disableProfile(myProfile2);
    // Should not be able to delete because the profile is by default enabled
    try {
        profileService.deleteProfile(myProfile);
        Assert.fail();
    } catch (ProfileConflictException e) {
    // expected
    }
    try {
        profileService.deleteAllProfiles(NamespaceId.DEFAULT);
        Assert.fail();
    } catch (ProfileConflictException e) {
        // expected and check profile 2 is not getting deleted
        Assert.assertEquals(profile2, profileService.getProfile(myProfile2));
    }
    // add assignment and disable it, deletion should also fail
    profileService.addProfileAssignment(myProfile, NamespaceId.DEFAULT);
    profileService.disableProfile(myProfile);
    try {
        profileService.deleteProfile(myProfile);
        Assert.fail();
    } catch (ProfileConflictException e) {
    // expected
    }
    try {
        profileService.deleteAllProfiles(NamespaceId.DEFAULT);
        Assert.fail();
    } catch (ProfileConflictException e) {
        // expected and check profile 2 is not getting deleted
        Assert.assertEquals(profile2, profileService.getProfile(myProfile2));
    }
    profileService.removeProfileAssignment(myProfile, NamespaceId.DEFAULT);
    // add an active record to DefaultStore, deletion should still fail
    Store store = getDefaultStore();
    ProgramId programId = NamespaceId.DEFAULT.app("myApp").workflow("myProgram");
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
    RunId runId = RunIds.generate(System.currentTimeMillis());
    ProgramRunId programRunId = programId.run(runId.getId());
    Map<String, String> systemArgs = Collections.singletonMap(SystemArguments.PROFILE_NAME, myProfile.getScopedName());
    int sourceId = 0;
    store.setProvisioning(programRunId, Collections.emptyMap(), systemArgs, AppFabricTestHelper.createSourceId(++sourceId), artifactId);
    store.setProvisioned(programRunId, 0, AppFabricTestHelper.createSourceId(++sourceId));
    store.setStart(programRunId, null, systemArgs, AppFabricTestHelper.createSourceId(++sourceId));
    try {
        profileService.deleteProfile(myProfile);
        Assert.fail();
    } catch (ProfileConflictException e) {
    // expected
    }
    try {
        profileService.deleteAllProfiles(NamespaceId.DEFAULT);
        Assert.fail();
    } catch (ProfileConflictException e) {
        // expected and check profile 2 is not getting deleted
        Assert.assertEquals(profile2, profileService.getProfile(myProfile2));
    }
    // set the run to stopped then deletion should work
    store.setStop(programRunId, System.currentTimeMillis() + 1000, ProgramController.State.ERROR.getRunStatus(), AppFabricTestHelper.createSourceId(++sourceId));
    // now profile deletion should succeed
    profileService.deleteProfile(myProfile);
    Assert.assertEquals(Collections.singletonList(profile2), profileService.getProfiles(NamespaceId.DEFAULT, false));
    profileService.saveProfile(myProfile, profile1);
    profileService.disableProfile(myProfile);
    profileService.deleteAllProfiles(NamespaceId.DEFAULT);
    Assert.assertEquals(Collections.emptyList(), profileService.getProfiles(NamespaceId.DEFAULT, false));
}

Example 10 with Store

use of io.cdap.cdap.app.store.Store in project cdap by cdapio.

the class LineageAdminTest method testSimpleLoopLineage.

@Test
public void testSimpleLoopLineage() {
    // Lineage for D1 -> P1 -> D2 -> P2 -> D3 -> P3 -> D4
    // |                 |
    // |                 V
    // |<-----------------
    // 
    TransactionRunner transactionRunner = getInjector().getInstance(TransactionRunner.class);
    LineageStoreReader lineageReader = new DefaultLineageStoreReader(transactionRunner);
    LineageWriter lineageWriter = new BasicLineageWriter(transactionRunner);
    Store store = getInjector().getInstance(Store.class);
    LineageAdmin lineageAdmin = new LineageAdmin(lineageReader, store);
    // Add access
    addRuns(store, run1, run2, run3, run4, run5);
    // It is okay to use current time here since access time is ignore during assertions
    lineageWriter.addAccess(run1, dataset1, AccessType.READ);
    lineageWriter.addAccess(run1, dataset2, AccessType.WRITE);
    lineageWriter.addAccess(run2, dataset2, AccessType.READ);
    lineageWriter.addAccess(run2, dataset1, AccessType.WRITE);
    lineageWriter.addAccess(run2, dataset3, AccessType.WRITE);
    lineageWriter.addAccess(run3, dataset3, AccessType.READ, null);
    lineageWriter.addAccess(run3, dataset4, AccessType.WRITE, null);
    Lineage expectedLineage = new Lineage(ImmutableSet.of(new Relation(dataset2, program1, AccessType.WRITE, twillRunId(run1)), new Relation(dataset1, program1, AccessType.READ, twillRunId(run1)), new Relation(dataset1, program2, AccessType.WRITE, twillRunId(run2)), new Relation(dataset2, program2, AccessType.READ, twillRunId(run2)), new Relation(dataset3, program2, AccessType.WRITE, twillRunId(run2)), new Relation(dataset4, program3, AccessType.WRITE, twillRunId(run3)), new Relation(dataset3, program3, AccessType.READ, twillRunId(run3))));
    // Lineage for D1
    Assert.assertEquals(expectedLineage, lineageAdmin.computeLineage(dataset1, 500, 20000, 100));
    // Lineage for D2
    Assert.assertEquals(expectedLineage, lineageAdmin.computeLineage(dataset2, 500, 20000, 100));
    // Lineage for D1 for one level D1 -> P1 -> D2 -> P2 -> D3
    // |                 |
    // |                 V
    // |<-----------------
    // 
    Lineage oneLevelLineage = lineageAdmin.computeLineage(dataset1, 500, 20000, 1);
    Assert.assertEquals(ImmutableSet.of(new Relation(dataset2, program1, AccessType.WRITE, twillRunId(run1)), new Relation(dataset1, program1, AccessType.READ, twillRunId(run1)), new Relation(dataset1, program2, AccessType.WRITE, twillRunId(run2)), new Relation(dataset2, program2, AccessType.READ, twillRunId(run2)), new Relation(dataset3, program2, AccessType.WRITE, twillRunId(run2))), oneLevelLineage.getRelations());
}