Search in sources :

Example 1 with MetadataStorage

use of io.cdap.cdap.spi.metadata.MetadataStorage in project cdap by caskdata.

the class MetadataSubscriberServiceTest method testProfileMetadata.

@Test
public void testProfileMetadata() throws Exception {
    Injector injector = getInjector();
    ApplicationSpecification appSpec = Specifications.from(new AppWithWorkflow());
    ApplicationId appId = NamespaceId.DEFAULT.app(appSpec.getName());
    ProgramId workflowId = appId.workflow("SampleWorkflow");
    ScheduleId scheduleId = appId.schedule("tsched1");
    // publish a creation of a schedule that will never exist
    // this tests that such a message is eventually discarded
    // note that for this test, we configure a fast retry strategy and a small number of retries
    // therefore this will cost only a few seconds delay
    publishBogusCreationEvent();
    // get the mds should be empty property since we haven't started the MetadataSubscriberService
    MetadataStorage mds = injector.getInstance(MetadataStorage.class);
    Assert.assertEquals(Collections.emptyMap(), mds.read(new Read(workflowId.toMetadataEntity())).getProperties());
    Assert.assertEquals(Collections.emptyMap(), mds.read(new Read(scheduleId.toMetadataEntity())).getProperties());
    // add a app with workflow to app meta store
    // note: since we bypass the app-fabric when adding this app, no ENTITY_CREATION message
    // will be published for the app (it happens in app lifecycle service). Therefore this
    // app must exist before assigning the profile for the namespace, otherwise the app's
    // programs will not receive the profile metadata.
    Store store = injector.getInstance(DefaultStore.class);
    store.addApplication(appId, appSpec);
    // set default namespace to use the profile, since now MetadataSubscriberService is not started,
    // it should not affect the mds
    PreferencesService preferencesService = injector.getInstance(PreferencesService.class);
    preferencesService.setProperties(NamespaceId.DEFAULT, Collections.singletonMap(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()));
    // add a schedule to schedule store
    ProgramScheduleService scheduleService = injector.getInstance(ProgramScheduleService.class);
    scheduleService.add(new ProgramSchedule("tsched1", "one time schedule", workflowId, Collections.emptyMap(), new TimeTrigger("* * ? * 1"), ImmutableList.of()));
    // add a new profile in default namespace
    ProfileService profileService = injector.getInstance(ProfileService.class);
    ProfileId myProfile = new ProfileId(NamespaceId.DEFAULT.getNamespace(), "MyProfile");
    Profile profile1 = new Profile("MyProfile", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), Profile.NATIVE.getProvisioner());
    profileService.saveProfile(myProfile, profile1);
    // add a second profile in default namespace
    ProfileId myProfile2 = new ProfileId(NamespaceId.DEFAULT.getNamespace(), "MyProfile2");
    Profile profile2 = new Profile("MyProfile2", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), Profile.NATIVE.getProvisioner());
    profileService.saveProfile(myProfile2, profile2);
    try {
        // Verify the workflow profile metadata is updated to default profile
        Tasks.waitFor(ProfileId.NATIVE.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // Verify the schedule profile metadata is updated to default profile
        Tasks.waitFor(ProfileId.NATIVE.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // set default namespace to use my profile
        preferencesService.setProperties(NamespaceId.DEFAULT, Collections.singletonMap(SystemArguments.PROFILE_NAME, "USER:MyProfile"));
        // Verify the workflow profile metadata is updated to my profile
        Tasks.waitFor(myProfile.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // Verify the schedule profile metadata is updated to my profile
        Tasks.waitFor(myProfile.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // set app level to use my profile 2
        preferencesService.setProperties(appId, Collections.singletonMap(SystemArguments.PROFILE_NAME, "USER:MyProfile2"));
        // set instance level to system profile
        preferencesService.setProperties(Collections.singletonMap(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()));
        // Verify the workflow profile metadata is updated to MyProfile2 which is at app level
        Tasks.waitFor(myProfile2.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // Verify the schedule profile metadata is updated to MyProfile2 which is at app level
        Tasks.waitFor(myProfile2.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // remove the preferences at instance level, should not affect the metadata
        preferencesService.deleteProperties();
        // Verify the workflow profile metadata is updated to default profile
        Tasks.waitFor(myProfile2.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // Verify the schedule profile metadata is updated to default profile
        Tasks.waitFor(myProfile2.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // remove app level pref should let the programs/schedules use ns level pref
        preferencesService.deleteProperties(appId);
        // Verify the workflow profile metadata is updated to MyProfile
        Tasks.waitFor(myProfile.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // Verify the schedule profile metadata is updated to MyProfile
        Tasks.waitFor(myProfile.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // remove ns level pref so no pref is there
        preferencesService.deleteProperties(NamespaceId.DEFAULT);
        // Verify the workflow profile metadata is updated to default profile
        Tasks.waitFor(ProfileId.NATIVE.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        // Verify the schedule profile metadata is updated to default profile
        Tasks.waitFor(ProfileId.NATIVE.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    } finally {
        // stop and clean up the store
        preferencesService.deleteProperties(NamespaceId.DEFAULT);
        preferencesService.deleteProperties();
        preferencesService.deleteProperties(appId);
        store.removeAll(NamespaceId.DEFAULT);
        scheduleService.delete(scheduleId);
        profileService.disableProfile(myProfile);
        profileService.disableProfile(myProfile2);
        profileService.deleteAllProfiles(myProfile.getNamespaceId());
        mds.apply(new MetadataMutation.Drop(workflowId.toMetadataEntity()), MutationOptions.DEFAULT);
        mds.apply(new MetadataMutation.Drop(scheduleId.toMetadataEntity()), MutationOptions.DEFAULT);
    }
}
Also used : ProfileId(io.cdap.cdap.proto.id.ProfileId) ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) TimeTrigger(io.cdap.cdap.internal.app.runtime.schedule.trigger.TimeTrigger) DefaultStore(io.cdap.cdap.internal.app.store.DefaultStore) Store(io.cdap.cdap.app.store.Store) ProgramId(io.cdap.cdap.proto.id.ProgramId) ScheduleId(io.cdap.cdap.proto.id.ScheduleId) AppWithWorkflow(io.cdap.cdap.AppWithWorkflow) Profile(io.cdap.cdap.proto.profile.Profile) PreferencesService(io.cdap.cdap.config.PreferencesService) Read(io.cdap.cdap.spi.metadata.Read) MetadataMutation(io.cdap.cdap.spi.metadata.MetadataMutation) ProfileService(io.cdap.cdap.internal.profile.ProfileService) ProgramSchedule(io.cdap.cdap.internal.app.runtime.schedule.ProgramSchedule) Injector(com.google.inject.Injector) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) ProgramScheduleService(io.cdap.cdap.scheduler.ProgramScheduleService) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) Test(org.junit.Test)

Example 2 with MetadataStorage

use of io.cdap.cdap.spi.metadata.MetadataStorage in project cdap by caskdata.

the class MetadataSubscriberServiceTest method testAppDeletionMessage.

@Test
public void testAppDeletionMessage() throws Exception {
    Injector injector = getInjector();
    // get the alert publisher
    CConfiguration cConf = injector.getInstance(CConfiguration.class);
    MessagingService messagingService = injector.getInstance(MessagingService.class);
    MultiThreadMessagingContext messagingContext = new MultiThreadMessagingContext(messagingService);
    AdminEventPublisher publisher = new AdminEventPublisher(cConf, messagingContext);
    // get the mds and put some workflow metadata in that, the publish of app deletion message should get the metadata
    // deleted
    MetadataStorage mds = injector.getInstance(MetadataStorage.class);
    // use an app with all program types to get all specification tested
    ApplicationId appId = NamespaceId.DEFAULT.app(AllProgramsApp.NAME);
    ProgramId workflowId = appId.workflow(AllProgramsApp.NoOpWorkflow.NAME);
    // generate an app spec from the application
    ApplicationSpecification appSpec = Specifications.from(new AllProgramsApp());
    // need to put metadata on workflow since we currently only set or delete workflow metadata
    mds.apply(new MetadataMutation.Update(workflowId.toMetadataEntity(), new Metadata(MetadataScope.SYSTEM, Collections.singletonMap("profile", ProfileId.NATIVE.getScopedName()))), MutationOptions.DEFAULT);
    Assert.assertEquals(ProfileId.NATIVE.getScopedName(), getProfileProperty(mds, workflowId));
    // publish app deletion message
    publisher.publishAppDeletion(appId, appSpec);
    // Verify the workflow profile metadata is removed because of the publish app deletion message
    Tasks.waitFor(true, () -> mds.read(new Read(workflowId.toMetadataEntity())).isEmpty(), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) AdminEventPublisher(io.cdap.cdap.internal.profile.AdminEventPublisher) Metadata(io.cdap.cdap.spi.metadata.Metadata) MultiThreadMessagingContext(io.cdap.cdap.messaging.context.MultiThreadMessagingContext) AllProgramsApp(io.cdap.cdap.AllProgramsApp) ProgramId(io.cdap.cdap.proto.id.ProgramId) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) MessagingService(io.cdap.cdap.messaging.MessagingService) CoreMessagingService(io.cdap.cdap.messaging.service.CoreMessagingService) Read(io.cdap.cdap.spi.metadata.Read) MetadataMutation(io.cdap.cdap.spi.metadata.MetadataMutation) Injector(com.google.inject.Injector) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) Test(org.junit.Test)

Example 3 with MetadataStorage

use of io.cdap.cdap.spi.metadata.MetadataStorage in project cdap by caskdata.

the class SystemMetadataWriterStageTest method setup.

@BeforeClass
public static void setup() {
    Injector injector = AppFabricTestHelper.getInjector();
    metadataStorage = injector.getInstance(MetadataStorage.class);
    metadataServiceClient = injector.getInstance(MetadataServiceClient.class);
    metadataSubscriber = injector.getInstance(MetadataSubscriberService.class);
    metadataSubscriber.startAndWait();
}
Also used : MetadataServiceClient(io.cdap.cdap.data2.metadata.writer.MetadataServiceClient) Injector(com.google.inject.Injector) MetadataSubscriberService(io.cdap.cdap.metadata.MetadataSubscriberService) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) BeforeClass(org.junit.BeforeClass)

Example 4 with MetadataStorage

use of io.cdap.cdap.spi.metadata.MetadataStorage in project cdap by caskdata.

the class ElasticsearchMetadataStorageTest method testScrollTimeout.

@Test
public void testScrollTimeout() throws IOException, InterruptedException {
    MetadataStorage mds = getMetadataStorage();
    MutationOptions options = MutationOptions.builder().setAsynchronous(false).build();
    List<MetadataRecord> records = IntStream.range(0, 20).boxed().map(i -> new MetadataRecord(MetadataEntity.ofDataset("ns" + i, "ds" + i), new Metadata(MetadataScope.USER, tags("tag", "t" + i), props("p", "v" + i)))).collect(Collectors.toList());
    mds.batch(records.stream().map(r -> new Update(r.getEntity(), r.getMetadata())).collect(Collectors.toList()), options);
    SearchRequest request = SearchRequest.of("t*").setCursorRequested(true).setLimit(5).build();
    SearchResponse response = mds.search(request);
    Assert.assertEquals(5, response.getResults().size());
    Assert.assertNotNull(response.getCursor());
    SearchRequest request2 = SearchRequest.of("t*").setCursor(response.getCursor()).build();
    SearchResponse response2 = mds.search(request2);
    Assert.assertEquals(5, response2.getResults().size());
    // it works despite a cursor and an offset (which is equal to the cursor's)
    SearchRequest request2a = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(5).build();
    SearchResponse response2a = mds.search(request2a);
    Assert.assertEquals(5, response2a.getOffset());
    Assert.assertEquals(5, response2a.getLimit());
    Assert.assertEquals(response2.getResults(), response2a.getResults());
    // it works despite a cursor and an offset (which is different from the cursor's)
    SearchRequest request2b = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(8).build();
    SearchResponse response2b = mds.search(request2b);
    Assert.assertEquals(5, response2b.getOffset());
    Assert.assertEquals(5, response2b.getLimit());
    Assert.assertEquals(response2.getResults(), response2b.getResults());
    // sleep 1 sec longer than the configured scroll timeout to invalidate cursor
    TimeUnit.SECONDS.sleep(3);
    SearchResponse response3 = mds.search(request2);
    Assert.assertEquals(response2.getResults(), response3.getResults());
    Assert.assertEquals(response2.getCursor(), response3.getCursor());
    // it works despite an expired cursor and an offset (which is different from the cursor's)
    SearchRequest request3a = SearchRequest.of("t*").setCursor(response.getCursor()).setOffset(8).build();
    SearchResponse response3a = mds.search(request3a);
    Assert.assertEquals(5, response3a.getOffset());
    Assert.assertEquals(5, response3a.getLimit());
    Assert.assertEquals(response2.getResults(), response3a.getResults());
    // create a nonsense cursor and search with that
    Cursor cursor = Cursor.fromString(response.getCursor());
    cursor = new Cursor(cursor, cursor.getOffset(), "nosuchcursor");
    SearchRequest request4 = SearchRequest.of("t*").setCursor(cursor.toString()).build();
    SearchResponse response4 = mds.search(request4);
    // compare only the results, not the entire response (the cursor is different)
    Assert.assertEquals(response2.getResults(), response4.getResults());
    // clean up
    mds.batch(records.stream().map(MetadataRecord::getEntity).map(Drop::new).collect(Collectors.toList()), options);
}
Also used : IntStream(java.util.stream.IntStream) Drop(io.cdap.cdap.spi.metadata.MetadataMutation.Drop) BeforeClass(org.junit.BeforeClass) LoggerFactory(org.slf4j.LoggerFactory) Random(java.util.Random) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) ImmutableList(com.google.common.collect.ImmutableList) ScopedNameOfKind(io.cdap.cdap.spi.metadata.ScopedNameOfKind) Metadata(io.cdap.cdap.spi.metadata.Metadata) Closeables(com.google.common.io.Closeables) MetadataStorageTest(io.cdap.cdap.spi.metadata.MetadataStorageTest) SearchRequest(io.cdap.cdap.spi.metadata.SearchRequest) MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) AfterClass(org.junit.AfterClass) ImmutableSet(com.google.common.collect.ImmutableSet) Logger(org.slf4j.Logger) Cursor(io.cdap.cdap.common.metadata.Cursor) Update(io.cdap.cdap.spi.metadata.MetadataMutation.Update) SearchResponse(io.cdap.cdap.spi.metadata.SearchResponse) Test(org.junit.Test) IOException(java.io.IOException) MetadataKind(io.cdap.cdap.spi.metadata.MetadataKind) ScopedName(io.cdap.cdap.spi.metadata.ScopedName) Collectors(java.util.stream.Collectors) TimeUnit(java.util.concurrent.TimeUnit) List(java.util.List) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope) MetadataRecord(io.cdap.cdap.spi.metadata.MetadataRecord) MutationOptions(io.cdap.cdap.spi.metadata.MutationOptions) Assert(org.junit.Assert) Collections(java.util.Collections) SConfiguration(io.cdap.cdap.common.conf.SConfiguration) SearchRequest(io.cdap.cdap.spi.metadata.SearchRequest) MutationOptions(io.cdap.cdap.spi.metadata.MutationOptions) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) Metadata(io.cdap.cdap.spi.metadata.Metadata) MetadataRecord(io.cdap.cdap.spi.metadata.MetadataRecord) Update(io.cdap.cdap.spi.metadata.MetadataMutation.Update) Cursor(io.cdap.cdap.common.metadata.Cursor) SearchResponse(io.cdap.cdap.spi.metadata.SearchResponse) Drop(io.cdap.cdap.spi.metadata.MetadataMutation.Drop) MetadataStorageTest(io.cdap.cdap.spi.metadata.MetadataStorageTest) Test(org.junit.Test)

Example 5 with MetadataStorage

use of io.cdap.cdap.spi.metadata.MetadataStorage in project cdap by caskdata.

the class ArtifactRepositoryTest method setup.

@BeforeClass
public static void setup() throws Exception {
    systemArtifactsDir1 = TMP_FOLDER.newFolder();
    systemArtifactsDir2 = TMP_FOLDER.newFolder();
    tmpDir = TMP_FOLDER.newFolder();
    cConf = CConfiguration.create();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TMP_FOLDER.newFolder().getAbsolutePath());
    cConf.set(Constants.AppFabric.SYSTEM_ARTIFACTS_DIR, systemArtifactsDir1.getAbsolutePath() + ";" + systemArtifactsDir2.getAbsolutePath());
    Injector injector = AppFabricTestHelper.getInjector(cConf);
    artifactRepository = injector.getInstance(ArtifactRepository.class);
    metadataStorage = injector.getInstance(MetadataStorage.class);
    appArtifactFile = createAppJar(PluginTestApp.class, new File(tmpDir, "PluginTest-1.0.0.jar"), createManifest(ManifestFields.EXPORT_PACKAGE, PluginTestRunnable.class.getPackage().getName()));
    metadataAdmin = injector.getInstance(MetadataAdmin.class);
}
Also used : PluginTestApp(io.cdap.cdap.internal.app.runtime.artifact.app.plugin.PluginTestApp) Injector(com.google.inject.Injector) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) File(java.io.File) MetadataAdmin(io.cdap.cdap.metadata.MetadataAdmin) BeforeClass(org.junit.BeforeClass)

Aggregations

MetadataStorage (io.cdap.cdap.spi.metadata.MetadataStorage)16 Injector (com.google.inject.Injector)7 Test (org.junit.Test)7 StructuredTableAdmin (io.cdap.cdap.spi.data.StructuredTableAdmin)6 MetricsCollectionService (io.cdap.cdap.api.metrics.MetricsCollectionService)5 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)5 MessagingService (io.cdap.cdap.messaging.MessagingService)5 MetadataService (io.cdap.cdap.metadata.MetadataService)5 MetadataSubscriberService (io.cdap.cdap.metadata.MetadataSubscriberService)5 Metadata (io.cdap.cdap.spi.metadata.Metadata)5 TransactionManager (org.apache.tephra.TransactionManager)5 BeforeClass (org.junit.BeforeClass)5 Service (com.google.common.util.concurrent.Service)4 AbstractModule (com.google.inject.AbstractModule)4 MetadataEntity (io.cdap.cdap.api.metadata.MetadataEntity)4 DatasetService (io.cdap.cdap.data2.datafabric.dataset.service.DatasetService)4 DatasetOpExecutorService (io.cdap.cdap.data2.datafabric.dataset.service.executor.DatasetOpExecutorService)4 AppFabricServer (io.cdap.cdap.internal.app.services.AppFabricServer)4 AppFabricTestModule (io.cdap.cdap.internal.guice.AppFabricTestModule)4 CoreSchedulerService (io.cdap.cdap.scheduler.CoreSchedulerService)4