use of io.cdap.cdap.spi.metadata.MetadataStorage in project cdap by caskdata.
the class MetadataSubscriberServiceTest method testProfileMetadata.
@Test
public void testProfileMetadata() throws Exception {
Injector injector = getInjector();
ApplicationSpecification appSpec = Specifications.from(new AppWithWorkflow());
ApplicationId appId = NamespaceId.DEFAULT.app(appSpec.getName());
ProgramId workflowId = appId.workflow("SampleWorkflow");
ScheduleId scheduleId = appId.schedule("tsched1");
// publish a creation of a schedule that will never exist
// this tests that such a message is eventually discarded
// note that for this test, we configure a fast retry strategy and a small number of retries
// therefore this will cost only a few seconds delay
publishBogusCreationEvent();
// get the mds should be empty property since we haven't started the MetadataSubscriberService
MetadataStorage mds = injector.getInstance(MetadataStorage.class);
Assert.assertEquals(Collections.emptyMap(), mds.read(new Read(workflowId.toMetadataEntity())).getProperties());
Assert.assertEquals(Collections.emptyMap(), mds.read(new Read(scheduleId.toMetadataEntity())).getProperties());
// add a app with workflow to app meta store
// note: since we bypass the app-fabric when adding this app, no ENTITY_CREATION message
// will be published for the app (it happens in app lifecycle service). Therefore this
// app must exist before assigning the profile for the namespace, otherwise the app's
// programs will not receive the profile metadata.
Store store = injector.getInstance(DefaultStore.class);
store.addApplication(appId, appSpec);
// set default namespace to use the profile, since now MetadataSubscriberService is not started,
// it should not affect the mds
PreferencesService preferencesService = injector.getInstance(PreferencesService.class);
preferencesService.setProperties(NamespaceId.DEFAULT, Collections.singletonMap(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()));
// add a schedule to schedule store
ProgramScheduleService scheduleService = injector.getInstance(ProgramScheduleService.class);
scheduleService.add(new ProgramSchedule("tsched1", "one time schedule", workflowId, Collections.emptyMap(), new TimeTrigger("* * ? * 1"), ImmutableList.of()));
// add a new profile in default namespace
ProfileService profileService = injector.getInstance(ProfileService.class);
ProfileId myProfile = new ProfileId(NamespaceId.DEFAULT.getNamespace(), "MyProfile");
Profile profile1 = new Profile("MyProfile", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), Profile.NATIVE.getProvisioner());
profileService.saveProfile(myProfile, profile1);
// add a second profile in default namespace
ProfileId myProfile2 = new ProfileId(NamespaceId.DEFAULT.getNamespace(), "MyProfile2");
Profile profile2 = new Profile("MyProfile2", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), Profile.NATIVE.getProvisioner());
profileService.saveProfile(myProfile2, profile2);
try {
// Verify the workflow profile metadata is updated to default profile
Tasks.waitFor(ProfileId.NATIVE.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Verify the schedule profile metadata is updated to default profile
Tasks.waitFor(ProfileId.NATIVE.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// set default namespace to use my profile
preferencesService.setProperties(NamespaceId.DEFAULT, Collections.singletonMap(SystemArguments.PROFILE_NAME, "USER:MyProfile"));
// Verify the workflow profile metadata is updated to my profile
Tasks.waitFor(myProfile.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Verify the schedule profile metadata is updated to my profile
Tasks.waitFor(myProfile.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// set app level to use my profile 2
preferencesService.setProperties(appId, Collections.singletonMap(SystemArguments.PROFILE_NAME, "USER:MyProfile2"));
// set instance level to system profile
preferencesService.setProperties(Collections.singletonMap(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()));
// Verify the workflow profile metadata is updated to MyProfile2 which is at app level
Tasks.waitFor(myProfile2.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Verify the schedule profile metadata is updated to MyProfile2 which is at app level
Tasks.waitFor(myProfile2.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// remove the preferences at instance level, should not affect the metadata
preferencesService.deleteProperties();
// Verify the workflow profile metadata is updated to default profile
Tasks.waitFor(myProfile2.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Verify the schedule profile metadata is updated to default profile
Tasks.waitFor(myProfile2.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// remove app level pref should let the programs/schedules use ns level pref
preferencesService.deleteProperties(appId);
// Verify the workflow profile metadata is updated to MyProfile
Tasks.waitFor(myProfile.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Verify the schedule profile metadata is updated to MyProfile
Tasks.waitFor(myProfile.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// remove ns level pref so no pref is there
preferencesService.deleteProperties(NamespaceId.DEFAULT);
// Verify the workflow profile metadata is updated to default profile
Tasks.waitFor(ProfileId.NATIVE.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Verify the schedule profile metadata is updated to default profile
Tasks.waitFor(ProfileId.NATIVE.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
} finally {
// stop and clean up the store
preferencesService.deleteProperties(NamespaceId.DEFAULT);
preferencesService.deleteProperties();
preferencesService.deleteProperties(appId);
store.removeAll(NamespaceId.DEFAULT);
scheduleService.delete(scheduleId);
profileService.disableProfile(myProfile);
profileService.disableProfile(myProfile2);
profileService.deleteAllProfiles(myProfile.getNamespaceId());
mds.apply(new MetadataMutation.Drop(workflowId.toMetadataEntity()), MutationOptions.DEFAULT);
mds.apply(new MetadataMutation.Drop(scheduleId.toMetadataEntity()), MutationOptions.DEFAULT);
}
}
use of io.cdap.cdap.spi.metadata.MetadataStorage in project cdap by caskdata.
the class MetadataSubscriberServiceTest method testAppDeletionMessage.
@Test
public void testAppDeletionMessage() throws Exception {
Injector injector = getInjector();
// get the alert publisher
CConfiguration cConf = injector.getInstance(CConfiguration.class);
MessagingService messagingService = injector.getInstance(MessagingService.class);
MultiThreadMessagingContext messagingContext = new MultiThreadMessagingContext(messagingService);
AdminEventPublisher publisher = new AdminEventPublisher(cConf, messagingContext);
// get the mds and put some workflow metadata in that, the publish of app deletion message should get the metadata
// deleted
MetadataStorage mds = injector.getInstance(MetadataStorage.class);
// use an app with all program types to get all specification tested
ApplicationId appId = NamespaceId.DEFAULT.app(AllProgramsApp.NAME);
ProgramId workflowId = appId.workflow(AllProgramsApp.NoOpWorkflow.NAME);
// generate an app spec from the application
ApplicationSpecification appSpec = Specifications.from(new AllProgramsApp());
// need to put metadata on workflow since we currently only set or delete workflow metadata
mds.apply(new MetadataMutation.Update(workflowId.toMetadataEntity(), new Metadata(MetadataScope.SYSTEM, Collections.singletonMap("profile", ProfileId.NATIVE.getScopedName()))), MutationOptions.DEFAULT);
Assert.assertEquals(ProfileId.NATIVE.getScopedName(), getProfileProperty(mds, workflowId));
// publish app deletion message
publisher.publishAppDeletion(appId, appSpec);
// Verify the workflow profile metadata is removed because of the publish app deletion message
Tasks.waitFor(true, () -> mds.read(new Read(workflowId.toMetadataEntity())).isEmpty(), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
}
use of io.cdap.cdap.spi.metadata.MetadataStorage in project cdap by caskdata.
the class SystemMetadataWriterStageTest method setup.
@BeforeClass
public static void setup() {
Injector injector = AppFabricTestHelper.getInjector();
metadataStorage = injector.getInstance(MetadataStorage.class);
metadataServiceClient = injector.getInstance(MetadataServiceClient.class);
metadataSubscriber = injector.getInstance(MetadataSubscriberService.class);
metadataSubscriber.startAndWait();
}
use of io.cdap.cdap.spi.metadata.MetadataStorage in project cdap by caskdata.
the class ElasticsearchMetadataStorageTest method testScrollTimeout.
@Test
public void testScrollTimeout() throws IOException, InterruptedException {
MetadataStorage mds = getMetadataStorage();
MutationOptions options = MutationOptions.builder().setAsynchronous(false).build();
List<MetadataRecord> records = IntStream.range(0, 20).boxed().map(i -> new MetadataRecord(MetadataEntity.ofDataset("ns" + i, "ds" + i), new Metadata(MetadataScope.USER, tags("tag", "t" + i), props("p", "v" + i)))).collect(Collectors.toList());
mds.batch(records.stream().map(r -> new Update(r.getEntity(), r.getMetadata())).collect(Collectors.toList()), options);
SearchRequest request = SearchRequest.of("t*").setCursorRequested(true).setLimit(5).build();
SearchResponse response = mds.search(request);
Assert.assertEquals(5, response.getResults().size());
Assert.assertNotNull(response.getCursor());
SearchRequest request2 = SearchRequest.of("t*").setCursor(response.getCursor()).build();
SearchResponse response2 = mds.search(request2);
Assert.assertEquals(5, response2.getResults().size());
// it works despite a cursor and an offset (which is equal to the cursor's)
SearchRequest request2a = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(5).build();
SearchResponse response2a = mds.search(request2a);
Assert.assertEquals(5, response2a.getOffset());
Assert.assertEquals(5, response2a.getLimit());
Assert.assertEquals(response2.getResults(), response2a.getResults());
// it works despite a cursor and an offset (which is different from the cursor's)
SearchRequest request2b = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(8).build();
SearchResponse response2b = mds.search(request2b);
Assert.assertEquals(5, response2b.getOffset());
Assert.assertEquals(5, response2b.getLimit());
Assert.assertEquals(response2.getResults(), response2b.getResults());
// sleep 1 sec longer than the configured scroll timeout to invalidate cursor
TimeUnit.SECONDS.sleep(3);
SearchResponse response3 = mds.search(request2);
Assert.assertEquals(response2.getResults(), response3.getResults());
Assert.assertEquals(response2.getCursor(), response3.getCursor());
// it works despite an expired cursor and an offset (which is different from the cursor's)
SearchRequest request3a = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(8).build();
SearchResponse response3a = mds.search(request3a);
Assert.assertEquals(5, response3a.getOffset());
Assert.assertEquals(5, response3a.getLimit());
Assert.assertEquals(response2.getResults(), response3a.getResults());
// create a nonsense cursor and search with that
Cursor cursor = Cursor.fromString(response.getCursor());
cursor = new Cursor(cursor, cursor.getOffset(), "nosuchcursor");
SearchRequest request4 = SearchRequest.of("t*").setCursor(cursor.toString()).build();
SearchResponse response4 = mds.search(request4);
// compare only the results, not the entire response (the cursor is different)
Assert.assertEquals(response2.getResults(), response4.getResults());
// clean up
mds.batch(records.stream().map(MetadataRecord::getEntity).map(Drop::new).collect(Collectors.toList()), options);
}
use of io.cdap.cdap.spi.metadata.MetadataStorage in project cdap by caskdata.
the class ArtifactRepositoryTest method setup.
@BeforeClass
public static void setup() throws Exception {
systemArtifactsDir1 = TMP_FOLDER.newFolder();
systemArtifactsDir2 = TMP_FOLDER.newFolder();
tmpDir = TMP_FOLDER.newFolder();
cConf = CConfiguration.create();
cConf.set(Constants.CFG_LOCAL_DATA_DIR, TMP_FOLDER.newFolder().getAbsolutePath());
cConf.set(Constants.AppFabric.SYSTEM_ARTIFACTS_DIR, systemArtifactsDir1.getAbsolutePath() + ";" + systemArtifactsDir2.getAbsolutePath());
Injector injector = AppFabricTestHelper.getInjector(cConf);
artifactRepository = injector.getInstance(ArtifactRepository.class);
metadataStorage = injector.getInstance(MetadataStorage.class);
appArtifactFile = createAppJar(PluginTestApp.class, new File(tmpDir, "PluginTest-1.0.0.jar"), createManifest(ManifestFields.EXPORT_PACKAGE, PluginTestRunnable.class.getPackage().getName()));
metadataAdmin = injector.getInstance(MetadataAdmin.class);
}
Aggregations