use of io.cdap.cdap.internal.profile.ProfileService in project cdap by caskdata.
the class MetadataSubscriberServiceTest method testProfileMetadata.
@Test
public void testProfileMetadata() throws Exception {
Injector injector = getInjector();
ApplicationSpecification appSpec = Specifications.from(new AppWithWorkflow());
ApplicationId appId = NamespaceId.DEFAULT.app(appSpec.getName());
ProgramId workflowId = appId.workflow("SampleWorkflow");
ScheduleId scheduleId = appId.schedule("tsched1");
// publish a creation of a schedule that will never exist
// this tests that such a message is eventually discarded
// note that for this test, we configure a fast retry strategy and a small number of retries
// therefore this will cost only a few seconds delay
publishBogusCreationEvent();
// get the mds should be empty property since we haven't started the MetadataSubscriberService
MetadataStorage mds = injector.getInstance(MetadataStorage.class);
Assert.assertEquals(Collections.emptyMap(), mds.read(new Read(workflowId.toMetadataEntity())).getProperties());
Assert.assertEquals(Collections.emptyMap(), mds.read(new Read(scheduleId.toMetadataEntity())).getProperties());
// add a app with workflow to app meta store
// note: since we bypass the app-fabric when adding this app, no ENTITY_CREATION message
// will be published for the app (it happens in app lifecycle service). Therefore this
// app must exist before assigning the profile for the namespace, otherwise the app's
// programs will not receive the profile metadata.
Store store = injector.getInstance(DefaultStore.class);
store.addApplication(appId, appSpec);
// set default namespace to use the profile, since now MetadataSubscriberService is not started,
// it should not affect the mds
PreferencesService preferencesService = injector.getInstance(PreferencesService.class);
preferencesService.setProperties(NamespaceId.DEFAULT, Collections.singletonMap(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()));
// add a schedule to schedule store
ProgramScheduleService scheduleService = injector.getInstance(ProgramScheduleService.class);
scheduleService.add(new ProgramSchedule("tsched1", "one time schedule", workflowId, Collections.emptyMap(), new TimeTrigger("* * ? * 1"), ImmutableList.of()));
// add a new profile in default namespace
ProfileService profileService = injector.getInstance(ProfileService.class);
ProfileId myProfile = new ProfileId(NamespaceId.DEFAULT.getNamespace(), "MyProfile");
Profile profile1 = new Profile("MyProfile", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), Profile.NATIVE.getProvisioner());
profileService.saveProfile(myProfile, profile1);
// add a second profile in default namespace
ProfileId myProfile2 = new ProfileId(NamespaceId.DEFAULT.getNamespace(), "MyProfile2");
Profile profile2 = new Profile("MyProfile2", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), Profile.NATIVE.getProvisioner());
profileService.saveProfile(myProfile2, profile2);
try {
// Verify the workflow profile metadata is updated to default profile
Tasks.waitFor(ProfileId.NATIVE.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Verify the schedule profile metadata is updated to default profile
Tasks.waitFor(ProfileId.NATIVE.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// set default namespace to use my profile
preferencesService.setProperties(NamespaceId.DEFAULT, Collections.singletonMap(SystemArguments.PROFILE_NAME, "USER:MyProfile"));
// Verify the workflow profile metadata is updated to my profile
Tasks.waitFor(myProfile.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Verify the schedule profile metadata is updated to my profile
Tasks.waitFor(myProfile.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// set app level to use my profile 2
preferencesService.setProperties(appId, Collections.singletonMap(SystemArguments.PROFILE_NAME, "USER:MyProfile2"));
// set instance level to system profile
preferencesService.setProperties(Collections.singletonMap(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()));
// Verify the workflow profile metadata is updated to MyProfile2 which is at app level
Tasks.waitFor(myProfile2.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Verify the schedule profile metadata is updated to MyProfile2 which is at app level
Tasks.waitFor(myProfile2.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// remove the preferences at instance level, should not affect the metadata
preferencesService.deleteProperties();
// Verify the workflow profile metadata is updated to default profile
Tasks.waitFor(myProfile2.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Verify the schedule profile metadata is updated to default profile
Tasks.waitFor(myProfile2.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// remove app level pref should let the programs/schedules use ns level pref
preferencesService.deleteProperties(appId);
// Verify the workflow profile metadata is updated to MyProfile
Tasks.waitFor(myProfile.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Verify the schedule profile metadata is updated to MyProfile
Tasks.waitFor(myProfile.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// remove ns level pref so no pref is there
preferencesService.deleteProperties(NamespaceId.DEFAULT);
// Verify the workflow profile metadata is updated to default profile
Tasks.waitFor(ProfileId.NATIVE.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Verify the schedule profile metadata is updated to default profile
Tasks.waitFor(ProfileId.NATIVE.getScopedName(), () -> getProfileProperty(mds, scheduleId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
} finally {
// stop and clean up the store
preferencesService.deleteProperties(NamespaceId.DEFAULT);
preferencesService.deleteProperties();
preferencesService.deleteProperties(appId);
store.removeAll(NamespaceId.DEFAULT);
scheduleService.delete(scheduleId);
profileService.disableProfile(myProfile);
profileService.disableProfile(myProfile2);
profileService.deleteAllProfiles(myProfile.getNamespaceId());
mds.apply(new MetadataMutation.Drop(workflowId.toMetadataEntity()), MutationOptions.DEFAULT);
mds.apply(new MetadataMutation.Drop(scheduleId.toMetadataEntity()), MutationOptions.DEFAULT);
}
}
use of io.cdap.cdap.internal.profile.ProfileService in project cdap by caskdata.
the class TestBase method initialize.
@BeforeClass
public static void initialize() throws Exception {
if (nestedStartCount++ > 0) {
return;
}
File localDataDir = TMP_FOLDER.newFolder();
cConf = createCConf(localDataDir);
CConfiguration previewCConf = createPreviewConf(cConf);
LevelDBTableService previewLevelDBTableService = new LevelDBTableService();
previewLevelDBTableService.setConfiguration(previewCConf);
// enable default services
File capabilityFolder = new File(localDataDir.toString(), "capability");
capabilityFolder.mkdir();
cConf.set(Constants.Capability.CONFIG_DIR, capabilityFolder.getAbsolutePath());
cConf.setInt(Constants.Capability.AUTO_INSTALL_THREADS, 5);
org.apache.hadoop.conf.Configuration hConf = new org.apache.hadoop.conf.Configuration();
hConf.addResource("mapred-site-local.xml");
hConf.reloadConfiguration();
hConf.set(Constants.CFG_LOCAL_DATA_DIR, localDataDir.getAbsolutePath());
hConf.set(Constants.AppFabric.OUTPUT_DIR, cConf.get(Constants.AppFabric.OUTPUT_DIR));
hConf.set("hadoop.tmp.dir", new File(localDataDir, cConf.get(Constants.AppFabric.TEMP_DIR)).getAbsolutePath());
// Windows specific requirements
if (OSDetector.isWindows()) {
File tmpDir = TMP_FOLDER.newFolder();
File binDir = new File(tmpDir, "bin");
Assert.assertTrue(binDir.mkdirs());
copyTempFile("hadoop.dll", tmpDir);
copyTempFile("winutils.exe", binDir);
System.setProperty("hadoop.home.dir", tmpDir.getAbsolutePath());
System.load(new File(tmpDir, "hadoop.dll").getAbsolutePath());
}
injector = Guice.createInjector(createDataFabricModule(), new TransactionExecutorModule(), new DataSetsModules().getStandaloneModules(), new DataSetServiceModules().getInMemoryModules(), new ConfigModule(cConf, hConf), RemoteAuthenticatorModules.getNoOpModule(), new IOModule(), new LocalLocationModule(), new InMemoryDiscoveryModule(), new AppFabricServiceRuntimeModule(cConf).getInMemoryModules(), new MonitorHandlerModule(false), new AuthenticationContextModules().getMasterModule(), new AuthorizationModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new ProgramRunnerRuntimeModule().getInMemoryModules(), new SecureStoreServerModule(), new MetadataReaderWriterModules().getInMemoryModules(), new MetadataServiceModule(), new AbstractModule() {
@Override
protected void configure() {
bind(MetricsManager.class).toProvider(MetricsManagerProvider.class);
}
}, new MetricsClientRuntimeModule().getInMemoryModules(), new LocalLogAppenderModule(), new LogReaderRuntimeModules().getInMemoryModules(), new ExploreRuntimeModule().getInMemoryModules(), new ExploreClientModule(), new MessagingServerRuntimeModule().getInMemoryModules(), new PreviewConfigModule(cConf, new Configuration(), SConfiguration.create()), new PreviewManagerModule(false), new PreviewRunnerManagerModule().getInMemoryModules(), new SupportBundleServiceModule(), new MockProvisionerModule(), new AbstractModule() {
@Override
protected void configure() {
install(new FactoryModuleBuilder().implement(ApplicationManager.class, DefaultApplicationManager.class).build(ApplicationManagerFactory.class));
install(new FactoryModuleBuilder().implement(ArtifactManager.class, DefaultArtifactManager.class).build(ArtifactManagerFactory.class));
bind(TemporaryFolder.class).toInstance(TMP_FOLDER);
bind(AuthorizationHandler.class).in(Scopes.SINGLETON);
// Needed by MonitorHandlerModuler
bind(TwillRunner.class).to(NoopTwillRunnerService.class);
bind(MetadataSubscriberService.class).in(Scopes.SINGLETON);
}
});
messagingService = injector.getInstance(MessagingService.class);
if (messagingService instanceof Service) {
((Service) messagingService).startAndWait();
}
txService = injector.getInstance(TransactionManager.class);
txService.startAndWait();
metadataSubscriberService = injector.getInstance(MetadataSubscriberService.class);
metadataStorage = injector.getInstance(MetadataStorage.class);
metadataAdmin = injector.getInstance(MetadataAdmin.class);
metadataStorage.createIndex();
metadataService = injector.getInstance(MetadataService.class);
metadataService.startAndWait();
// Define all StructuredTable before starting any services that need StructuredTable
StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class));
dsOpService = injector.getInstance(DatasetOpExecutorService.class);
dsOpService.startAndWait();
datasetService = injector.getInstance(DatasetService.class);
datasetService.startAndWait();
metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
metricsCollectionService.startAndWait();
if (cConf.getBoolean(Constants.Explore.EXPLORE_ENABLED)) {
exploreExecutorService = injector.getInstance(ExploreExecutorService.class);
exploreExecutorService.startAndWait();
// wait for explore service to be discoverable
DiscoveryServiceClient discoveryService = injector.getInstance(DiscoveryServiceClient.class);
EndpointStrategy endpointStrategy = new RandomEndpointStrategy(() -> discoveryService.discover(Constants.Service.EXPLORE_HTTP_USER_SERVICE));
Preconditions.checkNotNull(endpointStrategy.pick(5, TimeUnit.SECONDS), "%s service is not up after 5 seconds", Constants.Service.EXPLORE_HTTP_USER_SERVICE);
exploreClient = injector.getInstance(ExploreClient.class);
}
programScheduler = injector.getInstance(Scheduler.class);
if (programScheduler instanceof Service) {
((Service) programScheduler).startAndWait();
}
testManager = injector.getInstance(UnitTestManager.class);
metricsManager = injector.getInstance(MetricsManager.class);
accessControllerInstantiator = injector.getInstance(AccessControllerInstantiator.class);
// This is needed so the logged-in user can successfully create the default namespace
if (cConf.getBoolean(Constants.Security.Authorization.ENABLED)) {
String user = System.getProperty("user.name");
SecurityRequestContext.setUserId(user);
InstanceId instance = new InstanceId(cConf.get(Constants.INSTANCE_NAME));
Principal principal = new Principal(user, Principal.PrincipalType.USER);
accessControllerInstantiator.get().grant(Authorizable.fromEntityId(instance), principal, EnumSet.allOf(StandardPermission.class));
accessControllerInstantiator.get().grant(Authorizable.fromEntityId(NamespaceId.DEFAULT), principal, EnumSet.allOf(StandardPermission.class));
}
namespaceAdmin = injector.getInstance(NamespaceAdmin.class);
if (firstInit) {
// only create the default namespace on first test. if multiple tests are run in the same JVM,
// then any time after the first time, the default namespace already exists. That is because
// the namespaceAdmin.delete(Id.Namespace.DEFAULT) in finish() only clears the default namespace
// but does not remove it entirely
namespaceAdmin.create(NamespaceMeta.DEFAULT);
ProfileService profileService = injector.getInstance(ProfileService.class);
profileService.saveProfile(ProfileId.NATIVE, Profile.NATIVE);
}
secureStore = injector.getInstance(SecureStore.class);
secureStoreManager = injector.getInstance(SecureStoreManager.class);
messagingContext = new MultiThreadMessagingContext(messagingService);
firstInit = false;
previewHttpServer = injector.getInstance(PreviewHttpServer.class);
previewHttpServer.startAndWait();
fieldLineageAdmin = injector.getInstance(FieldLineageAdmin.class);
lineageAdmin = injector.getInstance(LineageAdmin.class);
metadataSubscriberService.startAndWait();
previewRunnerManager = injector.getInstance(PreviewRunnerManager.class);
if (previewRunnerManager instanceof Service) {
((Service) previewRunnerManager).startAndWait();
}
appFabricServer = injector.getInstance(AppFabricServer.class);
appFabricServer.startAndWait();
preferencesService = injector.getInstance(PreferencesService.class);
scheduler = injector.getInstance(Scheduler.class);
if (scheduler instanceof Service) {
((Service) scheduler).startAndWait();
}
if (scheduler instanceof CoreSchedulerService) {
((CoreSchedulerService) scheduler).waitUntilFunctional(10, TimeUnit.SECONDS);
}
supportBundleInternalService = injector.getInstance(SupportBundleInternalService.class);
supportBundleInternalService.startAndWait();
appFabricHealthCheckService = injector.getInstance(HealthCheckService.class);
appFabricHealthCheckService.helper(Constants.AppFabricHealthCheck.APP_FABRIC_HEALTH_CHECK_SERVICE, cConf, Constants.Service.MASTER_SERVICES_BIND_ADDRESS);
appFabricHealthCheckService.startAndWait();
}
use of io.cdap.cdap.internal.profile.ProfileService in project cdap by caskdata.
the class PreferencesServiceTest method testAddProfileInProperties.
@Test
public void testAddProfileInProperties() throws Exception {
PreferencesService prefStore = getInjector().getInstance(PreferencesService.class);
ProfileService profileStore = getInjector().getInstance(ProfileService.class);
// put a profile unrelated property should not affect the write
Map<String, String> expected = new HashMap<>();
expected.put("unRelatedKey", "unRelatedValue");
prefStore.setProperties(new ProgramId("myspace", "app", ProgramType.WORKFLOW, "prog"), expected);
Assert.assertEquals(expected, prefStore.getProperties(new ProgramId("myspace", "app", ProgramType.WORKFLOW, "prog")));
// put something related to profile
Map<String, String> profileMap = new HashMap<>();
profileMap.put(SystemArguments.PROFILE_NAME, "userProfile");
// this set call should fail since the profile does not exist
try {
prefStore.setProperties(new ProgramId("myspace", "app", ProgramType.WORKFLOW, "prog"), profileMap);
Assert.fail();
} catch (NotFoundException e) {
// expected
}
// the pref store should remain unchanged
Assert.assertEquals(expected, prefStore.getProperties(new ProgramId("myspace", "app", ProgramType.WORKFLOW, "prog")));
// add the profile and disable it
ProfileId profileId = new ProfileId("myspace", "userProfile");
profileStore.saveProfile(profileId, Profile.NATIVE);
profileStore.disableProfile(profileId);
// this set call should fail since the profile is disabled
try {
prefStore.setProperties(new ProgramId("myspace", "app", ProgramType.WORKFLOW, "prog"), profileMap);
Assert.fail();
} catch (ProfileConflictException e) {
// expected
}
// the pref store should remain unchanged
Assert.assertEquals(expected, prefStore.getProperties(new ProgramId("myspace", "app", ProgramType.WORKFLOW, "prog")));
// enable the profile
profileStore.enableProfile(profileId);
expected = profileMap;
prefStore.setProperties(new ProgramId("myspace", "app", ProgramType.WORKFLOW, "prog"), profileMap);
Map<String, String> properties = prefStore.getProperties(new ProgramId("myspace", "app", ProgramType.WORKFLOW, "prog"));
Assert.assertEquals(expected, properties);
prefStore.deleteProperties(new ProgramId("myspace", "app", ProgramType.WORKFLOW, "prog"));
profileStore.disableProfile(profileId);
profileStore.deleteProfile(profileId);
}
use of io.cdap.cdap.internal.profile.ProfileService in project cdap by caskdata.
the class PreferencesServiceTest method testProfileAssignment.
@Test
public void testProfileAssignment() throws Exception {
PreferencesService preferencesService = getInjector().getInstance(PreferencesService.class);
ProfileService profileService = getInjector().getInstance(ProfileService.class);
ProfileId myProfile = NamespaceId.DEFAULT.profile("myProfile");
profileService.saveProfile(myProfile, Profile.NATIVE);
// add properties with profile information
Map<String, String> prop = new HashMap<>();
prop.put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName());
ApplicationId myApp = NamespaceId.DEFAULT.app("myApp");
ProgramId myProgram = myApp.workflow("myProgram");
preferencesService.setProperties(prop);
preferencesService.setProperties(NamespaceId.DEFAULT, prop);
preferencesService.setProperties(myApp, prop);
preferencesService.setProperties(myProgram, prop);
// the assignment should be there for these entities
Set<EntityId> expected = new HashSet<>();
expected.add(new InstanceId(""));
expected.add(NamespaceId.DEFAULT);
expected.add(myApp);
expected.add(myProgram);
Assert.assertEquals(expected, profileService.getProfileAssignments(ProfileId.NATIVE));
// setting an empty property is actually deleting the assignment
prop.clear();
preferencesService.setProperties(myApp, prop);
expected.remove(myApp);
Assert.assertEquals(expected, profileService.getProfileAssignments(ProfileId.NATIVE));
// set my program to use a different profile, should update both profiles
prop.put(SystemArguments.PROFILE_NAME, myProfile.getScopedName());
preferencesService.setProperties(myProgram, prop);
expected.remove(myProgram);
Assert.assertEquals(expected, profileService.getProfileAssignments(ProfileId.NATIVE));
Assert.assertEquals(Collections.singleton(myProgram), profileService.getProfileAssignments(myProfile));
// delete all preferences
preferencesService.deleteProperties();
preferencesService.deleteProperties(NamespaceId.DEFAULT);
preferencesService.deleteProperties(myApp);
preferencesService.deleteProperties(myProgram);
Assert.assertEquals(Collections.emptySet(), profileService.getProfileAssignments(ProfileId.NATIVE));
Assert.assertEquals(Collections.emptySet(), profileService.getProfileAssignments(myProfile));
profileService.disableProfile(myProfile);
profileService.deleteProfile(myProfile);
}
use of io.cdap.cdap.internal.profile.ProfileService in project cdap by caskdata.
the class MetadataSubscriberServiceTest method testProfileMetadataWithNoProfilePreferences.
@Test
public void testProfileMetadataWithNoProfilePreferences() throws Exception {
Injector injector = getInjector();
// add a new profile in default namespace
ProfileService profileService = injector.getInstance(ProfileService.class);
ProfileId myProfile = new ProfileId(NamespaceId.DEFAULT.getNamespace(), "MyProfile");
Profile profile1 = new Profile("MyProfile", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), Profile.NATIVE.getProvisioner());
profileService.saveProfile(myProfile, profile1);
// add a app with workflow to app meta store
ApplicationSpecification appSpec = Specifications.from(new AppWithWorkflow());
ApplicationId appId = NamespaceId.DEFAULT.app(appSpec.getName());
ProgramId workflowId = appId.workflow("SampleWorkflow");
// get the metadata - should be empty since we haven't deployed the app
MetadataStorage mds = injector.getInstance(MetadataStorage.class);
Assert.assertEquals(Collections.emptyMap(), mds.read(new Read(workflowId.toMetadataEntity())).getProperties());
Store store = injector.getInstance(DefaultStore.class);
store.addApplication(appId, appSpec);
// set default namespace to use the profile, since now MetadataSubscriberService is not started,
// it should not affect the mds
PreferencesService preferencesService = injector.getInstance(PreferencesService.class);
preferencesService.setProperties(NamespaceId.DEFAULT, Collections.singletonMap(SystemArguments.PROFILE_NAME, "USER:MyProfile"));
try {
// Verify the workflow profile metadata is updated to my profile
Tasks.waitFor(myProfile.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Set the property without profile is a replacement of the preference, so it is same as deletion of the profile
preferencesService.setProperties(NamespaceId.DEFAULT, Collections.emptyMap());
// Verify the workflow profile metadata is updated to default profile
Tasks.waitFor(ProfileId.NATIVE.getScopedName(), () -> getProfileProperty(mds, workflowId), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
} finally {
// stop and clean up the store
preferencesService.deleteProperties(NamespaceId.DEFAULT);
store.removeAll(NamespaceId.DEFAULT);
profileService.disableProfile(myProfile);
profileService.deleteProfile(myProfile);
mds.apply(new MetadataMutation.Drop(workflowId.toMetadataEntity()), MutationOptions.DEFAULT);
}
}
Aggregations