use of io.cdap.cdap.security.auth.context.AuthenticationContextModules in project cdap by cdapio.
the class TransactionServiceClientTest method beforeClass.
@BeforeClass
public static void beforeClass() throws Exception {
HBaseTestingUtility hBaseTestingUtility = new HBaseTestingUtility();
hBaseTestingUtility.startMiniDFSCluster(1);
Configuration hConf = hBaseTestingUtility.getConfiguration();
hConf.setBoolean("fs.hdfs.impl.disable.cache", true);
zkServer = InMemoryZKServer.builder().build();
zkServer.startAndWait();
CConfiguration cConf = CConfiguration.create();
// tests should use the current user for HDFS
cConf.set(Constants.CFG_HDFS_USER, System.getProperty("user.name"));
cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath());
cConf.setBoolean(TxConstants.TransactionPruning.PRUNE_ENABLE, false);
// we want persisting for this test
cConf.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, true);
// getCommonConfiguration() sets up an hConf with tx service configuration.
// however, createTxService() will override these with defaults from the CConf.
// hence, we must pass in these settings when creating the tx service.
Configuration extraCConf = new Configuration();
extraCConf.clear();
extraCConf = getCommonConfiguration(extraCConf);
for (Map.Entry<String, String> entry : extraCConf) {
cConf.set(entry.getKey(), entry.getValue());
}
server = TransactionServiceTest.createTxService(zkServer.getConnectionStr(), Networks.getRandomPort(), hConf, tmpFolder.newFolder(), cConf);
server.startAndWait();
injector = Guice.createInjector(new ConfigModule(cConf, hConf), RemoteAuthenticatorModules.getNoOpModule(), new ZKClientModule(), new ZKDiscoveryModule(), new NonCustomLocationUnitTestModule(), new TransactionMetricsModule(), new DataFabricModules().getDistributedModules(), new AbstractModule() {
@Override
protected void configure() {
bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class);
bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
}
}, Modules.override(new DataSetsModules().getDistributedModules()).with(new AbstractModule() {
@Override
protected void configure() {
bind(MetadataStorage.class).to(NoopMetadataStorage.class);
}
}), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule());
zkClient = injector.getInstance(ZKClientService.class);
zkClient.startAndWait();
txStateStorage = injector.getInstance(TransactionStateStorage.class);
txStateStorage.startAndWait();
}
use of io.cdap.cdap.security.auth.context.AuthenticationContextModules in project cdap by cdapio.
the class TransactionServiceTest method testHA.
@Test(timeout = 30000)
public void testHA() throws Exception {
// NOTE: we play with blocking/nonblocking a lot below
// as until we integrate with "leader election" stuff, service blocks on start if it is not a leader
// TODO: fix this by integration with generic leader election stuff
CConfiguration cConf = CConfiguration.create();
// tests should use the current user for HDFS
cConf.set(Constants.CFG_HDFS_USER, System.getProperty("user.name"));
cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath());
Injector injector = Guice.createInjector(new ConfigModule(cConf), RemoteAuthenticatorModules.getNoOpModule(), new ZKClientModule(), new ZKDiscoveryModule(), new NonCustomLocationUnitTestModule(), new TransactionMetricsModule(), new AbstractModule() {
@Override
protected void configure() {
bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class);
bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
}
}, new DataFabricModules().getDistributedModules(), Modules.override(new DataSetsModules().getDistributedModules()).with(new AbstractModule() {
@Override
protected void configure() {
bind(MetadataStorage.class).to(NoopMetadataStorage.class);
}
}), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule());
ZKClientService zkClient = injector.getInstance(ZKClientService.class);
zkClient.startAndWait();
try {
final Table table = createTable("myTable");
// tx service client
// NOTE: we can init it earlier than we start services, it should pick them up when they are available
TransactionSystemClient txClient = injector.getInstance(TransactionSystemClient.class);
TransactionExecutor txExecutor = new DefaultTransactionExecutor(txClient, ImmutableList.of((TransactionAware) table));
// starting tx service, tx client can pick it up
TransactionService first = createTxService(zkServer.getConnectionStr(), Networks.getRandomPort(), hConf, tmpFolder.newFolder());
first.startAndWait();
Assert.assertNotNull(txClient.startShort());
verifyGetAndPut(table, txExecutor, null, "val1");
// starting another tx service should not hurt
TransactionService second = createTxService(zkServer.getConnectionStr(), Networks.getRandomPort(), hConf, tmpFolder.newFolder());
// NOTE: we don't have to wait for start as client should pick it up anyways, but we do wait to ensure
// the case with two active is handled well
second.startAndWait();
// wait for affect a bit
TimeUnit.SECONDS.sleep(1);
Assert.assertNotNull(txClient.startShort());
verifyGetAndPut(table, txExecutor, "val1", "val2");
// shutting down the first one is fine: we have another one to pick up the leader role
first.stopAndWait();
Assert.assertNotNull(txClient.startShort());
verifyGetAndPut(table, txExecutor, "val2", "val3");
// doing same trick again to failover to the third one
TransactionService third = createTxService(zkServer.getConnectionStr(), Networks.getRandomPort(), hConf, tmpFolder.newFolder());
// NOTE: we don't have to wait for start as client should pick it up anyways
third.start();
// stopping second one
second.stopAndWait();
Assert.assertNotNull(txClient.startShort());
verifyGetAndPut(table, txExecutor, "val3", "val4");
// releasing resources
third.stop();
} finally {
try {
dropTable("myTable");
} finally {
zkClient.stopAndWait();
}
}
}
use of io.cdap.cdap.security.auth.context.AuthenticationContextModules in project cdap by cdapio.
the class AbstractServiceMain method init.
@Override
public final void init(String[] args) throws Exception {
LOG.info("Initializing master service class {}", getClass().getName());
// System wide setup
Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler());
// Intercept JUL loggers
SLF4JBridgeHandler.removeHandlersForRootLogger();
SLF4JBridgeHandler.install();
TypeToken<?> type = TypeToken.of(getClass()).resolveType(AbstractServiceMain.class.getTypeParameters()[0]);
T options = (T) type.getRawType().newInstance();
OptionsParser.init(options, args, getClass().getSimpleName(), ProjectInfo.getVersion().toString(), System.out);
CConfiguration cConf = CConfiguration.create();
SecurityUtil.loginForMasterService(cConf);
SConfiguration sConf = SConfiguration.create();
if (options.getExtraConfPath() != null) {
cConf.addResource(new File(options.getExtraConfPath(), "cdap-site.xml").toURI().toURL());
sConf.addResource(new File(options.getExtraConfPath(), "cdap-security.xml").toURI().toURL());
}
cConf = updateCConf(cConf);
Configuration hConf = new Configuration();
masterEnv = MasterEnvironments.setMasterEnvironment(MasterEnvironments.create(cConf, options.getEnvProvider()));
MasterEnvironmentContext masterEnvContext = MasterEnvironments.createContext(cConf, hConf, masterEnv.getName());
masterEnv.initialize(masterEnvContext);
List<Module> modules = new ArrayList<>();
modules.add(new ConfigModule(cConf, hConf, sConf));
modules.add(RemoteAuthenticatorModules.getDefaultModule());
modules.add(new PreviewConfigModule(cConf, hConf, sConf));
modules.add(new IOModule());
modules.add(new MetricsClientRuntimeModule().getDistributedModules());
modules.add(new AbstractModule() {
@Override
protected void configure() {
bind(DiscoveryService.class).toProvider(new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceSupplier()));
bind(DiscoveryServiceClient.class).toProvider(new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceClientSupplier()));
}
});
modules.add(getLogAppenderModule());
CoreSecurityModule coreSecurityModule = CoreSecurityRuntimeModule.getDistributedModule(cConf);
modules.add(coreSecurityModule);
if (coreSecurityModule.requiresZKClient()) {
modules.add(new ZKClientModule());
}
modules.add(new AuthenticationContextModules().getMasterModule());
modules.addAll(getServiceModules(masterEnv, options, cConf));
injector = Guice.createInjector(modules);
// Initialize logging context
LogAppenderInitializer logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
closeableResources.add(logAppenderInitializer);
logAppenderInitializer.initialize();
Optional.ofNullable(getLoggingContext(options)).ifPresent(LoggingContextAccessor::setLoggingContext);
// Add Services
services.add(injector.getInstance(MetricsCollectionService.class));
addServices(injector, services, closeableResources, masterEnv, masterEnvContext, options);
// Optionally get the storage provider. It is for destroy() method to close it on shutdown.
Binding<StorageProvider> storageBinding = injector.getExistingBinding(Key.get(StorageProvider.class));
if (storageBinding != null) {
storageProvider = storageBinding.getProvider().get();
}
LOG.info("Service {} initialized", getClass().getName());
}
use of io.cdap.cdap.security.auth.context.AuthenticationContextModules in project cdap by cdapio.
the class DatasetServiceTestBase method initializeAndStartService.
protected static void initializeAndStartService(CConfiguration cConf) throws Exception {
// TODO: this whole method is a mess. Streamline it!
injector = Guice.createInjector(new ConfigModule(cConf), RemoteAuthenticatorModules.getNoOpModule(), new InMemoryDiscoveryModule(), new NonCustomLocationUnitTestModule(), new NamespaceAdminTestModule(), new SystemDatasetRuntimeModule().getInMemoryModules(), new TransactionInMemoryModule(), new AuthorizationTestModule(), new StorageModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getMasterModule(), new AbstractModule() {
@Override
protected void configure() {
bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class).in(Singleton.class);
bind(DatasetDefinitionRegistryFactory.class).to(DefaultDatasetDefinitionRegistryFactory.class).in(Scopes.SINGLETON);
// through the injector, we only need RemoteDatasetFramework in these tests
bind(RemoteDatasetFramework.class);
bind(OwnerStore.class).to(InMemoryOwnerStore.class);
bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
}
});
AccessEnforcer authEnforcer = injector.getInstance(AccessEnforcer.class);
AuthenticationContext authenticationContext = injector.getInstance(AuthenticationContext.class);
transactionRunner = injector.getInstance(TransactionRunner.class);
DiscoveryService discoveryService = injector.getInstance(DiscoveryService.class);
discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class);
dsFramework = injector.getInstance(RemoteDatasetFramework.class);
// Tx Manager to support working with datasets
txManager = injector.getInstance(TransactionManager.class);
txManager.startAndWait();
StructuredTableAdmin structuredTableAdmin = injector.getInstance(StructuredTableAdmin.class);
StoreDefinition.createAllTables(structuredTableAdmin);
TransactionSystemClient txSystemClient = injector.getInstance(TransactionSystemClient.class);
TransactionSystemClientService txSystemClientService = new DelegatingTransactionSystemClientService(txSystemClient);
NamespacePathLocator namespacePathLocator = injector.getInstance(NamespacePathLocator.class);
SystemDatasetInstantiatorFactory datasetInstantiatorFactory = new SystemDatasetInstantiatorFactory(locationFactory, dsFramework, cConf);
// ok to pass null, since the impersonator won't actually be called, if kerberos security is not enabled
Impersonator impersonator = new DefaultImpersonator(cConf, null);
DatasetAdminService datasetAdminService = new DatasetAdminService(dsFramework, cConf, locationFactory, datasetInstantiatorFactory, impersonator);
ImmutableSet<HttpHandler> handlers = ImmutableSet.<HttpHandler>of(new DatasetAdminOpHTTPHandler(datasetAdminService));
CommonNettyHttpServiceFactory commonNettyHttpServiceFactory = injector.getInstance(CommonNettyHttpServiceFactory.class);
opExecutorService = new DatasetOpExecutorService(cConf, SConfiguration.create(), discoveryService, commonNettyHttpServiceFactory, handlers);
opExecutorService.startAndWait();
Map<String, DatasetModule> defaultModules = injector.getInstance(Key.get(new TypeLiteral<Map<String, DatasetModule>>() {
}, Constants.Dataset.Manager.DefaultDatasetModules.class));
ImmutableMap<String, DatasetModule> modules = ImmutableMap.<String, DatasetModule>builder().putAll(defaultModules).build();
registryFactory = injector.getInstance(DatasetDefinitionRegistryFactory.class);
inMemoryDatasetFramework = new InMemoryDatasetFramework(registryFactory, modules);
DiscoveryExploreClient exploreClient = new DiscoveryExploreClient(discoveryServiceClient, authenticationContext);
ExploreFacade exploreFacade = new ExploreFacade(exploreClient, cConf);
namespaceAdmin = injector.getInstance(NamespaceAdmin.class);
namespaceAdmin.create(NamespaceMeta.DEFAULT);
ownerAdmin = injector.getInstance(OwnerAdmin.class);
NamespaceQueryAdmin namespaceQueryAdmin = injector.getInstance(NamespaceQueryAdmin.class);
DatasetTypeManager typeManager = new DatasetTypeManager(cConf, locationFactory, impersonator, transactionRunner);
DatasetOpExecutor opExecutor = new InMemoryDatasetOpExecutor(dsFramework);
DatasetInstanceManager instanceManager = new DatasetInstanceManager(transactionRunner);
DatasetTypeService noAuthTypeService = new DefaultDatasetTypeService(typeManager, namespaceAdmin, namespacePathLocator, cConf, impersonator, txSystemClientService, transactionRunner, defaultModules);
DatasetTypeService typeService = new AuthorizationDatasetTypeService(noAuthTypeService, authEnforcer, authenticationContext);
instanceService = new DatasetInstanceService(typeService, noAuthTypeService, instanceManager, opExecutor, exploreFacade, namespaceQueryAdmin, ownerAdmin, authEnforcer, authenticationContext, new NoOpMetadataServiceClient());
service = new DatasetService(cConf, SConfiguration.create(), discoveryService, discoveryServiceClient, commonNettyHttpServiceFactory, new HashSet<>(), typeService, instanceService);
// Start dataset service, wait for it to be discoverable
service.startAndWait();
waitForService(Constants.Service.DATASET_EXECUTOR);
waitForService(Constants.Service.DATASET_MANAGER);
// this usually happens while creating a namespace, however not doing that in data fabric tests
Locations.mkdirsIfNotExists(namespacePathLocator.get(NamespaceId.DEFAULT));
}
use of io.cdap.cdap.security.auth.context.AuthenticationContextModules in project cdap by cdapio.
the class DatasetOpExecutorServiceTest method setUp.
@Before
public void setUp() throws Exception {
Configuration hConf = new Configuration();
CConfiguration cConf = CConfiguration.create();
File datasetDir = new File(TMP_FOLDER.newFolder(), "datasetUser");
Assert.assertTrue(datasetDir.mkdirs());
cConf.set(Constants.Dataset.Manager.OUTPUT_DIR, datasetDir.getAbsolutePath());
cConf.set(Constants.Service.MASTER_SERVICES_BIND_ADDRESS, "localhost");
cConf.set(Constants.Dataset.Executor.ADDRESS, "localhost");
cConf.setInt(Constants.Dataset.Executor.PORT, Networks.getRandomPort());
Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), RemoteAuthenticatorModules.getNoOpModule(), new IOModule(), new ZKClientModule(), new KafkaClientModule(), new InMemoryDiscoveryModule(), new NonCustomLocationUnitTestModule(), new DataFabricModules().getInMemoryModules(), new DataSetsModules().getStandaloneModules(), new DataSetServiceModules().getInMemoryModules(), new TransactionMetricsModule(), new ExploreClientModule(), new NamespaceAdminTestModule(), new AuthenticationContextModules().getMasterModule(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AbstractModule() {
@Override
protected void configure() {
bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
bind(MetadataServiceClient.class).to(NoOpMetadataServiceClient.class);
}
});
txManager = injector.getInstance(TransactionManager.class);
txManager.startAndWait();
StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class));
dsOpExecService = injector.getInstance(DatasetOpExecutorService.class);
dsOpExecService.startAndWait();
managerService = injector.getInstance(DatasetService.class);
managerService.startAndWait();
dsFramework = injector.getInstance(DatasetFramework.class);
// find host
DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class);
endpointStrategy = new RandomEndpointStrategy(() -> discoveryClient.discover(Constants.Service.DATASET_MANAGER));
namespaceAdmin = injector.getInstance(NamespaceAdmin.class);
namespaceAdmin.create(NamespaceMeta.DEFAULT);
namespaceAdmin.create(new NamespaceMeta.Builder().setName(bob.getParent()).build());
}
Aggregations