Examples with DatasetModule co.cask.cdap.api.dataset.module.DatasetModule used on opensource projects

Example 6 with DatasetModule

use of co.cask.cdap.api.dataset.module.DatasetModule in project cdap by caskdata.

the class DatasetServiceTestBase method initializeAndStartService.

protected static void initializeAndStartService(CConfiguration cConf) throws Exception {
    // TODO: this whole method is a mess. Streamline it!
    injector = Guice.createInjector(new ConfigModule(cConf), new DiscoveryRuntimeModule().getInMemoryModules(), new NonCustomLocationUnitTestModule().getModule(), new NamespaceClientRuntimeModule().getInMemoryModules(), new SystemDatasetRuntimeModule().getInMemoryModules(), new TransactionInMemoryModule(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getMasterModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class).in(Singleton.class);
            install(new FactoryModuleBuilder().implement(DatasetDefinitionRegistry.class, DefaultDatasetDefinitionRegistry.class).build(DatasetDefinitionRegistryFactory.class));
            // through the injector, we only need RemoteDatasetFramework in these tests
            bind(RemoteDatasetFramework.class);
            bind(OwnerStore.class).to(InMemoryOwnerStore.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    });
    AuthorizationEnforcer authEnforcer = injector.getInstance(AuthorizationEnforcer.class);
    AuthenticationContext authenticationContext = injector.getInstance(AuthenticationContext.class);
    DiscoveryService discoveryService = injector.getInstance(DiscoveryService.class);
    discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class);
    dsFramework = injector.getInstance(RemoteDatasetFramework.class);
    // Tx Manager to support working with datasets
    txManager = injector.getInstance(TransactionManager.class);
    txManager.startAndWait();
    TransactionSystemClient txSystemClient = injector.getInstance(TransactionSystemClient.class);
    TransactionSystemClientService txSystemClientService = new DelegatingTransactionSystemClientService(txSystemClient);
    NamespacedLocationFactory namespacedLocationFactory = injector.getInstance(NamespacedLocationFactory.class);
    SystemDatasetInstantiatorFactory datasetInstantiatorFactory = new SystemDatasetInstantiatorFactory(locationFactory, dsFramework, cConf);
    // ok to pass null, since the impersonator won't actually be called, if kerberos security is not enabled
    Impersonator impersonator = new DefaultImpersonator(cConf, null);
    DatasetAdminService datasetAdminService = new DatasetAdminService(dsFramework, cConf, locationFactory, datasetInstantiatorFactory, new NoOpMetadataStore(), impersonator);
    ImmutableSet<HttpHandler> handlers = ImmutableSet.<HttpHandler>of(new DatasetAdminOpHTTPHandler(datasetAdminService));
    MetricsCollectionService metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
    opExecutorService = new DatasetOpExecutorService(cConf, discoveryService, metricsCollectionService, handlers);
    opExecutorService.startAndWait();
    Map<String, DatasetModule> defaultModules = injector.getInstance(Key.get(new TypeLiteral<Map<String, DatasetModule>>() {
    }, Constants.Dataset.Manager.DefaultDatasetModules.class));
    ImmutableMap<String, DatasetModule> modules = ImmutableMap.<String, DatasetModule>builder().putAll(defaultModules).putAll(DatasetMetaTableUtil.getModules()).build();
    registryFactory = injector.getInstance(DatasetDefinitionRegistryFactory.class);
    inMemoryDatasetFramework = new InMemoryDatasetFramework(registryFactory, modules);
    DiscoveryExploreClient exploreClient = new DiscoveryExploreClient(discoveryServiceClient, authenticationContext);
    ExploreFacade exploreFacade = new ExploreFacade(exploreClient, cConf);
    namespaceAdmin = injector.getInstance(NamespaceAdmin.class);
    namespaceAdmin.create(NamespaceMeta.DEFAULT);
    ownerAdmin = injector.getInstance(OwnerAdmin.class);
    NamespaceQueryAdmin namespaceQueryAdmin = injector.getInstance(NamespaceQueryAdmin.class);
    TransactionExecutorFactory txExecutorFactory = new DynamicTransactionExecutorFactory(txSystemClient);
    DatasetTypeManager typeManager = new DatasetTypeManager(cConf, locationFactory, txSystemClientService, txExecutorFactory, inMemoryDatasetFramework, impersonator);
    DatasetOpExecutor opExecutor = new InMemoryDatasetOpExecutor(dsFramework);
    DatasetInstanceManager instanceManager = new DatasetInstanceManager(txSystemClientService, txExecutorFactory, inMemoryDatasetFramework);
    PrivilegesManager privilegesManager = injector.getInstance(PrivilegesManager.class);
    DatasetTypeService typeService = new DatasetTypeService(typeManager, namespaceAdmin, namespacedLocationFactory, authEnforcer, privilegesManager, authenticationContext, cConf, impersonator, txSystemClientService, inMemoryDatasetFramework, txExecutorFactory, defaultModules);
    instanceService = new DatasetInstanceService(typeService, instanceManager, opExecutor, exploreFacade, namespaceQueryAdmin, ownerAdmin, authEnforcer, privilegesManager, authenticationContext);
    service = new DatasetService(cConf, discoveryService, discoveryServiceClient, metricsCollectionService, opExecutor, new HashSet<DatasetMetricsReporter>(), typeService, instanceService);
    // Start dataset service, wait for it to be discoverable
    service.startAndWait();
    waitForService(Constants.Service.DATASET_EXECUTOR);
    waitForService(Constants.Service.DATASET_MANAGER);
    // this usually happens while creating a namespace, however not doing that in data fabric tests
    Locations.mkdirsIfNotExists(namespacedLocationFactory.get(NamespaceId.DEFAULT));
}

Example 7 with DatasetModule

use of co.cask.cdap.api.dataset.module.DatasetModule in project cdap by caskdata.

the class DefaultPreviewManager method createPreviewInjector.

/**
   * Create injector for the given application id.
   */
@VisibleForTesting
Injector createPreviewInjector(ApplicationId applicationId) throws IOException {
    CConfiguration previewCConf = CConfiguration.copy(cConf);
    java.nio.file.Path previewDirPath = Paths.get(cConf.get(Constants.CFG_LOCAL_DATA_DIR), "preview").toAbsolutePath();
    Files.createDirectories(previewDirPath);
    java.nio.file.Path previewDir = Files.createDirectories(Paths.get(previewDirPath.toAbsolutePath().toString(), applicationId.getApplication()));
    previewCConf.set(Constants.CFG_LOCAL_DATA_DIR, previewDir.toString());
    Configuration previewHConf = new Configuration(hConf);
    previewHConf.set(Constants.CFG_LOCAL_DATA_DIR, previewDir.toString());
    previewCConf.setIfUnset(Constants.CFG_DATA_LEVELDB_DIR, previewDir.toString());
    previewCConf.setBoolean(Constants.Explore.EXPLORE_ENABLED, false);
    return Guice.createInjector(new ConfigModule(previewCConf, previewHConf), new IOModule(), new AuthenticationContextModules().getMasterModule(), new SecurityModules().getStandaloneModules(), new PreviewSecureStoreModule(secureStore), new PreviewStreamAdminModule(streamAdmin), new PreviewDiscoveryRuntimeModule(discoveryService), new LocationRuntimeModule().getStandaloneModules(), new ConfigStoreModule().getStandaloneModule(), new PreviewRunnerModule(artifactRepository, artifactStore, authorizerInstantiator, authorizationEnforcer, privilegesManager, streamCoordinatorClient, preferencesStore), new ProgramRunnerRuntimeModule().getStandaloneModules(), new PreviewDataModules().getDataFabricModule(transactionManager), new PreviewDataModules().getDataSetsModule(datasetFramework), new DataSetServiceModules().getStandaloneModules(), new MetricsClientRuntimeModule().getStandaloneModules(), new LoggingModules().getStandaloneModules(), new NamespaceStoreModule().getStandaloneModules(), new MessagingServerRuntimeModule().getInMemoryModules(), new AbstractModule() {

        @Override
        protected void configure() {
            // Bind system datasets defined in App-fabric.
            // Have to do it here as public binding, instead of inside PreviewRunnerModule due to Guice 3
            // doesn't support exporting multi-binder from private module
            MapBinder<String, DatasetModule> datasetModuleBinder = MapBinder.newMapBinder(binder(), String.class, DatasetModule.class, Constants.Dataset.Manager.DefaultDatasetModules.class);
            datasetModuleBinder.addBinding("app-fabric").toInstance(new AppFabricDatasetModule());
        }

        @Provides
        @Named(Constants.Service.MASTER_SERVICES_BIND_ADDRESS)
        @SuppressWarnings("unused")
        public InetAddress providesHostname(CConfiguration cConf) {
            String address = cConf.get(Constants.Preview.ADDRESS);
            return Networks.resolve(address, new InetSocketAddress("localhost", 0).getAddress());
        }
    });
}

Example 8 with DatasetModule

use of co.cask.cdap.api.dataset.module.DatasetModule in project cdap by caskdata.

the class DatasetDefinitionRegistries method register.

public static void register(String moduleClassName, @Nullable ClassLoader classLoader, DatasetDefinitionRegistry registry) throws ClassNotFoundException, IllegalAccessException, InstantiationException, TypeConflictException {
    ClassLoader systemClassLoader = DatasetDefinitionRegistries.class.getClassLoader();
    // Either uses the given classloader or the system one
    ClassLoader moduleClassLoader = Objects.firstNonNull(classLoader, systemClassLoader);
    Class<? extends DatasetModule> moduleClass;
    try {
        moduleClass = loadClass(moduleClassLoader, moduleClassName);
    } catch (ClassNotFoundException e) {
        // If failed to load from the given classloader (if not null), try to load it from system classloader
        if (classLoader == null || classLoader.equals(systemClassLoader)) {
            throw e;
        }
        try {
            moduleClass = loadClass(systemClassLoader, moduleClassName);
        } catch (ClassNotFoundException e2) {
            e.addSuppressed(e2);
            throw e;
        }
    }
    DatasetModule module = DatasetModules.getDatasetModule(moduleClass);
    module.register(registry);
}

Example 9 with DatasetModule

use of co.cask.cdap.api.dataset.module.DatasetModule in project cdap by caskdata.

the class DatasetTypeService method deployDefaultModules.

private void deployDefaultModules() throws Exception {
    List<DatasetModuleId> toGrant = new ArrayList<>();
    // adding default modules to be available in dataset manager service
    for (Map.Entry<String, DatasetModule> module : defaultModules.entrySet()) {
        try {
            // NOTE: we assume default modules are always in classpath, hence passing null for jar location
            // NOTE: we add default modules in the system namespace
            DatasetModuleId defaultModule = NamespaceId.SYSTEM.datasetModule(module.getKey());
            typeManager.addModule(defaultModule, module.getValue().getClass().getName(), null, false);
            toGrant.add(defaultModule);
        } catch (DatasetModuleConflictException e) {
            // perfectly fine: we need to add default modules only the very first time service is started
            LOG.debug("Not adding {} module: it already exists", module.getKey());
        } catch (Throwable th) {
            LOG.error("Failed to add {} module. Aborting.", module.getKey(), th);
            throw Throwables.propagate(th);
        }
    }
    long startTime = System.currentTimeMillis();
    LOG.trace("Granting all privileges for {} default dataset modules. ", toGrant.size());
    for (DatasetModuleId defaultModule : toGrant) {
        grantAllPrivilegesOnModule(defaultModule, authenticationContext.getPrincipal());
    }
    long doneTime = System.currentTimeMillis();
    float elapsedSeconds = doneTime == startTime ? 0.0F : ((float) doneTime - startTime) / 1000;
    LOG.debug("Granting all privileges for {} default dataset modules took {} seconds.", toGrant.size(), elapsedSeconds);
}

Example 10 with DatasetModule

use of co.cask.cdap.api.dataset.module.DatasetModule in project cdap by caskdata.

the class DatasetTypeService method getExtensionModules.

private Map<String, DatasetModule> getExtensionModules(CConfiguration cConf) {
    Map<String, DatasetModule> modules = new LinkedHashMap<>();
    String moduleStr = cConf.get(Constants.Dataset.Extensions.MODULES);
    if (moduleStr != null) {
        for (String moduleName : Splitter.on(',').omitEmptyStrings().split(moduleStr)) {
            // create DatasetModule object
            try {
                Class tableModuleClass = Class.forName(moduleName);
                DatasetModule module = (DatasetModule) tableModuleClass.newInstance();
                modules.put(moduleName, module);
            } catch (ClassCastException | ClassNotFoundException | InstantiationException | IllegalAccessException ex) {
                LOG.error("Failed to add {} extension module: {}", moduleName, ex.toString());
            }
        }
    }
    return modules;
}