use of io.cdap.cdap.api.dataset.module.DatasetModule in project cdap by caskdata.
the class DatasetModulesDeployer method loadAndDeployModule.
private void loadAndDeployModule(ClassLoader artifactClassLoader, String className, final Location jarLocation, String moduleName, NamespaceId namespaceId, String authorizingUser) throws Exception {
// note: using app class loader to load module class
@SuppressWarnings("unchecked") Class<Dataset> clazz = (Class<Dataset>) artifactClassLoader.loadClass(className);
try {
// note: we can deploy module or create module from Dataset class
// note: it seems dangerous to instantiate dataset module here, but this will be fine when we move deploy into
// isolated user's environment (e.g. separate yarn container)
final DatasetModuleId moduleId = namespaceId.datasetModule(moduleName);
final DatasetModule module;
if (DatasetModule.class.isAssignableFrom(clazz)) {
module = (DatasetModule) clazz.newInstance();
} else if (Dataset.class.isAssignableFrom(clazz)) {
if (systemDatasetFramework.hasSystemType(clazz.getName())) {
return;
}
final DatasetTypeId typeId = namespaceId.datasetType(clazz.getName());
boolean hasType = AuthorizationUtil.authorizeAs(authorizingUser, new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
return datasetFramework.hasType(typeId);
}
});
if (hasType && !allowDatasetUncheckedUpgrade) {
return;
}
module = new SingleTypeModule(clazz);
} else {
throw new IllegalArgumentException(String.format("Cannot use class %s to add dataset module: it must be of type DatasetModule or Dataset", clazz.getName()));
}
LOG.info("Adding module: {}", clazz.getName());
AuthorizationUtil.authorizeAs(authorizingUser, new Callable<Void>() {
@Override
public Void call() throws Exception {
datasetFramework.addModule(moduleId, module, jarLocation);
return null;
}
});
} catch (ModuleConflictException e) {
LOG.info("Conflict while deploying module {}: {}", moduleName, e.getMessage());
throw e;
}
}
use of io.cdap.cdap.api.dataset.module.DatasetModule in project cdap by caskdata.
the class DatasetServiceTestBase method initializeAndStartService.
protected static void initializeAndStartService(CConfiguration cConf) throws Exception {
// TODO: this whole method is a mess. Streamline it!
injector = Guice.createInjector(new ConfigModule(cConf), RemoteAuthenticatorModules.getNoOpModule(), new InMemoryDiscoveryModule(), new NonCustomLocationUnitTestModule(), new NamespaceAdminTestModule(), new SystemDatasetRuntimeModule().getInMemoryModules(), new TransactionInMemoryModule(), new AuthorizationTestModule(), new StorageModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getMasterModule(), new AbstractModule() {
@Override
protected void configure() {
bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class).in(Singleton.class);
bind(DatasetDefinitionRegistryFactory.class).to(DefaultDatasetDefinitionRegistryFactory.class).in(Scopes.SINGLETON);
// through the injector, we only need RemoteDatasetFramework in these tests
bind(RemoteDatasetFramework.class);
bind(OwnerStore.class).to(InMemoryOwnerStore.class);
bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
}
});
AccessEnforcer authEnforcer = injector.getInstance(AccessEnforcer.class);
AuthenticationContext authenticationContext = injector.getInstance(AuthenticationContext.class);
transactionRunner = injector.getInstance(TransactionRunner.class);
DiscoveryService discoveryService = injector.getInstance(DiscoveryService.class);
discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class);
dsFramework = injector.getInstance(RemoteDatasetFramework.class);
// Tx Manager to support working with datasets
txManager = injector.getInstance(TransactionManager.class);
txManager.startAndWait();
StructuredTableAdmin structuredTableAdmin = injector.getInstance(StructuredTableAdmin.class);
StoreDefinition.createAllTables(structuredTableAdmin);
TransactionSystemClient txSystemClient = injector.getInstance(TransactionSystemClient.class);
TransactionSystemClientService txSystemClientService = new DelegatingTransactionSystemClientService(txSystemClient);
NamespacePathLocator namespacePathLocator = injector.getInstance(NamespacePathLocator.class);
SystemDatasetInstantiatorFactory datasetInstantiatorFactory = new SystemDatasetInstantiatorFactory(locationFactory, dsFramework, cConf);
// ok to pass null, since the impersonator won't actually be called, if kerberos security is not enabled
Impersonator impersonator = new DefaultImpersonator(cConf, null);
DatasetAdminService datasetAdminService = new DatasetAdminService(dsFramework, cConf, locationFactory, datasetInstantiatorFactory, impersonator);
ImmutableSet<HttpHandler> handlers = ImmutableSet.<HttpHandler>of(new DatasetAdminOpHTTPHandler(datasetAdminService));
MetricsCollectionService metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
opExecutorService = new DatasetOpExecutorService(cConf, SConfiguration.create(), discoveryService, metricsCollectionService, handlers);
opExecutorService.startAndWait();
Map<String, DatasetModule> defaultModules = injector.getInstance(Key.get(new TypeLiteral<Map<String, DatasetModule>>() {
}, Constants.Dataset.Manager.DefaultDatasetModules.class));
ImmutableMap<String, DatasetModule> modules = ImmutableMap.<String, DatasetModule>builder().putAll(defaultModules).build();
registryFactory = injector.getInstance(DatasetDefinitionRegistryFactory.class);
inMemoryDatasetFramework = new InMemoryDatasetFramework(registryFactory, modules);
DiscoveryExploreClient exploreClient = new DiscoveryExploreClient(discoveryServiceClient, authenticationContext);
ExploreFacade exploreFacade = new ExploreFacade(exploreClient, cConf);
namespaceAdmin = injector.getInstance(NamespaceAdmin.class);
namespaceAdmin.create(NamespaceMeta.DEFAULT);
ownerAdmin = injector.getInstance(OwnerAdmin.class);
NamespaceQueryAdmin namespaceQueryAdmin = injector.getInstance(NamespaceQueryAdmin.class);
DatasetTypeManager typeManager = new DatasetTypeManager(cConf, locationFactory, impersonator, transactionRunner);
DatasetOpExecutor opExecutor = new InMemoryDatasetOpExecutor(dsFramework);
DatasetInstanceManager instanceManager = new DatasetInstanceManager(transactionRunner);
DatasetTypeService noAuthTypeService = new DefaultDatasetTypeService(typeManager, namespaceAdmin, namespacePathLocator, cConf, impersonator, txSystemClientService, transactionRunner, defaultModules);
DatasetTypeService typeService = new AuthorizationDatasetTypeService(noAuthTypeService, authEnforcer, authenticationContext);
instanceService = new DatasetInstanceService(typeService, noAuthTypeService, instanceManager, opExecutor, exploreFacade, namespaceQueryAdmin, ownerAdmin, authEnforcer, authenticationContext, new NoOpMetadataServiceClient());
service = new DatasetService(cConf, SConfiguration.create(), discoveryService, discoveryServiceClient, metricsCollectionService, new HashSet<>(), typeService, instanceService);
// Start dataset service, wait for it to be discoverable
service.startAndWait();
waitForService(Constants.Service.DATASET_EXECUTOR);
waitForService(Constants.Service.DATASET_MANAGER);
// this usually happens while creating a namespace, however not doing that in data fabric tests
Locations.mkdirsIfNotExists(namespacePathLocator.get(NamespaceId.DEFAULT));
}
use of io.cdap.cdap.api.dataset.module.DatasetModule in project cdap by caskdata.
the class SystemDatasetRuntimeModule method getStandaloneModules.
@Override
public Module getStandaloneModules() {
return new AbstractModule() {
@Override
protected void configure() {
MapBinder<String, DatasetModule> mapBinder = MapBinder.newMapBinder(binder(), String.class, DatasetModule.class, Constants.Dataset.Manager.DefaultDatasetModules.class);
// NOTE: order is important due to dependencies between modules
mapBinder.addBinding("orderedTable-leveldb").toInstance(new LevelDBTableModule());
mapBinder.addBinding("metricsTable-leveldb").toInstance(new LevelDBMetricsTableModule());
bindDefaultModules(mapBinder);
bind(String.class).annotatedWith(Names.named(Constants.Dataset.TABLE_TYPE)).toInstance("table");
bind(DatasetDefinition.class).annotatedWith(Names.named(Constants.Dataset.TABLE_TYPE)).to(LevelDBTableDefinition.class);
bind(String.class).annotatedWith(Names.named(Constants.Dataset.TABLE_TYPE_NO_TX)).toInstance("table-no-tx");
bind(DatasetDefinition.class).annotatedWith(Names.named(Constants.Dataset.TABLE_TYPE_NO_TX)).to(LevelDBMetricsTableDefinition.class);
// Direct binding for the Metrics table definition such that metrics system doesn't need to go through
// dataset service to get metrics table.
bind(new TypeLiteral<DatasetDefinition<MetricsTable, DatasetAdmin>>() {
}).toInstance(new LevelDBMetricsTableDefinition(MetricsTable.class.getName()));
}
};
}
use of io.cdap.cdap.api.dataset.module.DatasetModule in project cdap by caskdata.
the class DefaultDatasetTypeService method deployExtensionModules.
private void deployExtensionModules() {
// adding any defined extension modules to be available in dataset manager service
for (Map.Entry<String, DatasetModule> module : extensionModules.entrySet()) {
try {
// NOTE: we assume extension modules are always in classpath, hence passing null for jar location
// NOTE: we add extension modules in the system namespace
DatasetModuleId theModule = NamespaceId.SYSTEM.datasetModule(module.getKey());
typeManager.addModule(theModule, module.getValue().getClass().getName(), null, false);
} catch (DatasetModuleConflictException e) {
// perfectly fine: we need to add the modules only the very first time service is started
LOG.debug("Not adding {} extension module: it already exists", module.getKey());
} catch (Throwable th) {
LOG.error("Failed to add {} extension module. Aborting.", module.getKey(), th);
throw Throwables.propagate(th);
}
}
}
use of io.cdap.cdap.api.dataset.module.DatasetModule in project cdap by caskdata.
the class DefaultDatasetTypeService method getExtensionModules.
private Map<String, DatasetModule> getExtensionModules(CConfiguration cConf) {
Map<String, DatasetModule> modules = new LinkedHashMap<>();
String moduleStr = cConf.get(Constants.Dataset.Extensions.MODULES);
if (moduleStr != null) {
for (String moduleName : Splitter.on(',').omitEmptyStrings().split(moduleStr)) {
// create DatasetModule object
try {
Class tableModuleClass = Class.forName(moduleName);
DatasetModule module = (DatasetModule) tableModuleClass.newInstance();
modules.put(moduleName, module);
} catch (ClassCastException | ClassNotFoundException | InstantiationException | IllegalAccessException ex) {
LOG.error("Failed to add {} extension module: {}", moduleName, ex.toString());
}
}
}
return modules;
}
Aggregations