Search in sources :

Example 16 with DataFabricModules

use of co.cask.cdap.data.runtime.DataFabricModules in project cdap by caskdata.

the class JobQueueDebugger method createInjector.

private static Injector createInjector() throws Exception {
    CConfiguration cConf = CConfiguration.create();
    if (cConf.getBoolean(Constants.Security.Authorization.ENABLED)) {
        System.out.println(String.format("Disabling authorization for %s.", JobQueueDebugger.class.getSimpleName()));
        cConf.setBoolean(Constants.Security.Authorization.ENABLED, false);
    }
    // Note: login has to happen before any objects that need Kerberos credentials are instantiated.
    SecurityUtil.loginForMasterService(cConf);
    return Guice.createInjector(new ConfigModule(cConf, HBaseConfiguration.create()), new IOModule(), new ZKClientModule(), new LocationRuntimeModule().getDistributedModules(), new DiscoveryRuntimeModule().getDistributedModules(), new ViewAdminModules().getDistributedModules(), new StreamAdminModules().getDistributedModules(), new NotificationFeedClientModule(), new TwillModule(), new ExploreClientModule(), new DataFabricModules().getDistributedModules(), new ServiceStoreModules().getDistributedModules(), new DataSetsModules().getDistributedModules(), new AppFabricServiceRuntimeModule().getDistributedModules(), new ProgramRunnerRuntimeModule().getDistributedModules(), new SystemDatasetRuntimeModule().getDistributedModules(), new NotificationServiceRuntimeModule().getDistributedModules(), new MetricsClientRuntimeModule().getDistributedModules(), new MetricsStoreModule(), new KafkaClientModule(), new NamespaceStoreModule().getDistributedModules(), new AuthorizationModule(), new AuthorizationEnforcementModule().getMasterModule(), new SecureStoreModules().getDistributedModules(), new MessagingClientModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(HBaseTableUtil.class).toProvider(HBaseTableUtilFactory.class);
            bind(Store.class).annotatedWith(Names.named("defaultStore")).to(DefaultStore.class).in(Singleton.class);
            // This is needed because the LocalApplicationManager
            // expects a dsframework injection named datasetMDS
            bind(DatasetFramework.class).annotatedWith(Names.named("datasetMDS")).to(DatasetFramework.class).in(Singleton.class);
        }
    });
}
Also used : IOModule(co.cask.cdap.common.guice.IOModule) MessagingClientModule(co.cask.cdap.messaging.guice.MessagingClientModule) MetricsStoreModule(co.cask.cdap.metrics.guice.MetricsStoreModule) ConfigModule(co.cask.cdap.common.guice.ConfigModule) NamespaceStoreModule(co.cask.cdap.store.guice.NamespaceStoreModule) NotificationServiceRuntimeModule(co.cask.cdap.notifications.guice.NotificationServiceRuntimeModule) MetricsClientRuntimeModule(co.cask.cdap.metrics.guice.MetricsClientRuntimeModule) ViewAdminModules(co.cask.cdap.data.view.ViewAdminModules) ZKClientModule(co.cask.cdap.common.guice.ZKClientModule) KafkaClientModule(co.cask.cdap.common.guice.KafkaClientModule) HBaseTableUtilFactory(co.cask.cdap.data2.util.hbase.HBaseTableUtilFactory) SystemDatasetRuntimeModule(co.cask.cdap.data.runtime.SystemDatasetRuntimeModule) DiscoveryRuntimeModule(co.cask.cdap.common.guice.DiscoveryRuntimeModule) AuthorizationModule(co.cask.cdap.app.guice.AuthorizationModule) TwillModule(co.cask.cdap.app.guice.TwillModule) DataSetsModules(co.cask.cdap.data.runtime.DataSetsModules) SecureStoreModules(co.cask.cdap.security.guice.SecureStoreModules) LocationRuntimeModule(co.cask.cdap.common.guice.LocationRuntimeModule) CConfiguration(co.cask.cdap.common.conf.CConfiguration) AbstractModule(com.google.inject.AbstractModule) StreamAdminModules(co.cask.cdap.data.stream.StreamAdminModules) ProgramRunnerRuntimeModule(co.cask.cdap.app.guice.ProgramRunnerRuntimeModule) ExploreClientModule(co.cask.cdap.explore.guice.ExploreClientModule) Singleton(com.google.inject.Singleton) NotificationFeedClientModule(co.cask.cdap.notifications.feeds.client.NotificationFeedClientModule) DataFabricModules(co.cask.cdap.data.runtime.DataFabricModules) ServiceStoreModules(co.cask.cdap.app.guice.ServiceStoreModules) AppFabricServiceRuntimeModule(co.cask.cdap.app.guice.AppFabricServiceRuntimeModule) AuthorizationEnforcementModule(co.cask.cdap.security.authorization.AuthorizationEnforcementModule)

Example 17 with DataFabricModules

use of co.cask.cdap.data.runtime.DataFabricModules in project cdap by caskdata.

the class LocalLogAppenderResilientTest method testResilientLogging.

@Test
public void testResilientLogging() throws Exception {
    Configuration hConf = new Configuration();
    CConfiguration cConf = CConfiguration.create();
    File datasetDir = new File(tmpFolder.newFolder(), "datasetUser");
    //noinspection ResultOfMethodCallIgnored
    datasetDir.mkdirs();
    cConf.set(Constants.Dataset.Manager.OUTPUT_DIR, datasetDir.getAbsolutePath());
    cConf.set(Constants.Service.MASTER_SERVICES_BIND_ADDRESS, "localhost");
    cConf.set(Constants.Dataset.Executor.ADDRESS, "localhost");
    cConf.setInt(Constants.Dataset.Executor.PORT, Networks.getRandomPort());
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath());
    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), new IOModule(), new ZKClientModule(), new KafkaClientModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new NonCustomLocationUnitTestModule().getModule(), new DataFabricModules().getInMemoryModules(), new DataSetsModules().getStandaloneModules(), new DataSetServiceModules().getInMemoryModules(), new TransactionMetricsModule(), new ExploreClientModule(), new LoggingModules().getInMemoryModules(), new NamespaceClientRuntimeModule().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getMasterModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(NoOpOwnerAdmin.class);
        }
    });
    TransactionManager txManager = injector.getInstance(TransactionManager.class);
    txManager.startAndWait();
    DatasetOpExecutorService opExecutorService = injector.getInstance(DatasetOpExecutorService.class);
    opExecutorService.startAndWait();
    // Start the logging before starting the service.
    LoggingContextAccessor.setLoggingContext(new FlowletLoggingContext("TRL_ACCT_1", "APP_1", "FLOW_1", "FLOWLET_1", "RUN", "INSTANCE"));
    String logBaseDir = "trl-log/log_files_" + new Random(System.currentTimeMillis()).nextLong();
    cConf.set(LoggingConfiguration.LOG_BASE_DIR, logBaseDir);
    cConf.setInt(LoggingConfiguration.LOG_MAX_FILE_SIZE_BYTES, 20 * 1024);
    final LogAppender appender = injector.getInstance(LocalLogAppender.class);
    new LogAppenderInitializer(appender).initialize("TestResilientLogging");
    int failureMsgCount = 3;
    final CountDownLatch failureLatch = new CountDownLatch(failureMsgCount);
    LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
    loggerContext.getStatusManager().add(new StatusListener() {

        @Override
        public void addStatusEvent(Status status) {
            if (status.getLevel() != Status.ERROR || status.getOrigin() != appender) {
                return;
            }
            Throwable cause = status.getThrowable();
            if (cause != null) {
                Throwable rootCause = Throwables.getRootCause(cause);
                if (rootCause instanceof ServiceUnavailableException) {
                    String serviceName = ((ServiceUnavailableException) rootCause).getServiceName();
                    if (Constants.Service.DATASET_MANAGER.equals(serviceName)) {
                        failureLatch.countDown();
                    }
                }
            }
        }
    });
    Logger logger = LoggerFactory.getLogger("TestResilientLogging");
    for (int i = 0; i < failureMsgCount; ++i) {
        Exception e1 = new Exception("Test Exception1");
        Exception e2 = new Exception("Test Exception2", e1);
        logger.warn("Test log message " + i + " {} {}", "arg1", "arg2", e2);
    }
    // Wait for the three failure to append to happen
    // The wait time has to be > 3 seconds because DatasetServiceClient has 1 second timeout on discovery
    failureLatch.await(5, TimeUnit.SECONDS);
    // Start dataset service, wait for it to be discoverable
    DatasetService dsService = injector.getInstance(DatasetService.class);
    dsService.startAndWait();
    final CountDownLatch startLatch = new CountDownLatch(1);
    DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class);
    discoveryClient.discover(Constants.Service.DATASET_MANAGER).watchChanges(new ServiceDiscovered.ChangeListener() {

        @Override
        public void onChange(ServiceDiscovered serviceDiscovered) {
            if (!Iterables.isEmpty(serviceDiscovered)) {
                startLatch.countDown();
            }
        }
    }, Threads.SAME_THREAD_EXECUTOR);
    startLatch.await(5, TimeUnit.SECONDS);
    // Do some more logging after the service is started.
    for (int i = 5; i < 10; ++i) {
        Exception e1 = new Exception("Test Exception1");
        Exception e2 = new Exception("Test Exception2", e1);
        logger.warn("Test log message " + i + " {} {}", "arg1", "arg2", e2);
    }
    appender.stop();
    // Verify - we should have at least 5 events.
    LoggingContext loggingContext = new FlowletLoggingContext("TRL_ACCT_1", "APP_1", "FLOW_1", "", "RUN", "INSTANCE");
    FileLogReader logTail = injector.getInstance(FileLogReader.class);
    LoggingTester.LogCallback logCallback1 = new LoggingTester.LogCallback();
    logTail.getLogPrev(loggingContext, ReadRange.LATEST, 10, Filter.EMPTY_FILTER, logCallback1);
    List<LogEvent> allEvents = logCallback1.getEvents();
    Assert.assertTrue(allEvents.toString(), allEvents.size() >= 5);
    // Finally - stop all services
    Services.chainStop(dsService, opExecutorService, txManager);
}
Also used : NoOpOwnerAdmin(co.cask.cdap.security.impersonation.NoOpOwnerAdmin) DataSetServiceModules(co.cask.cdap.data.runtime.DataSetServiceModules) DiscoveryServiceClient(org.apache.twill.discovery.DiscoveryServiceClient) CConfiguration(co.cask.cdap.common.conf.CConfiguration) Configuration(org.apache.hadoop.conf.Configuration) LoggingConfiguration(co.cask.cdap.logging.LoggingConfiguration) LocalLogAppender(co.cask.cdap.logging.framework.local.LocalLogAppender) DatasetService(co.cask.cdap.data2.datafabric.dataset.service.DatasetService) TransactionMetricsModule(co.cask.cdap.data.runtime.TransactionMetricsModule) FileLogReader(co.cask.cdap.logging.read.FileLogReader) Random(java.util.Random) Injector(com.google.inject.Injector) KafkaClientModule(co.cask.cdap.common.guice.KafkaClientModule) FlowletLoggingContext(co.cask.cdap.logging.context.FlowletLoggingContext) DiscoveryRuntimeModule(co.cask.cdap.common.guice.DiscoveryRuntimeModule) Status(ch.qos.logback.core.status.Status) LogEvent(co.cask.cdap.logging.read.LogEvent) AuthenticationContextModules(co.cask.cdap.security.auth.context.AuthenticationContextModules) ServiceDiscovered(org.apache.twill.discovery.ServiceDiscovered) NonCustomLocationUnitTestModule(co.cask.cdap.common.guice.NonCustomLocationUnitTestModule) CConfiguration(co.cask.cdap.common.conf.CConfiguration) ExploreClientModule(co.cask.cdap.explore.guice.ExploreClientModule) TransactionManager(org.apache.tephra.TransactionManager) File(java.io.File) DataFabricModules(co.cask.cdap.data.runtime.DataFabricModules) AuthorizationEnforcementModule(co.cask.cdap.security.authorization.AuthorizationEnforcementModule) IOModule(co.cask.cdap.common.guice.IOModule) NamespaceClientRuntimeModule(co.cask.cdap.common.namespace.guice.NamespaceClientRuntimeModule) LoggingContext(co.cask.cdap.common.logging.LoggingContext) FlowletLoggingContext(co.cask.cdap.logging.context.FlowletLoggingContext) ConfigModule(co.cask.cdap.common.guice.ConfigModule) ServiceUnavailableException(co.cask.cdap.common.ServiceUnavailableException) Logger(org.slf4j.Logger) ZKClientModule(co.cask.cdap.common.guice.ZKClientModule) UnsupportedUGIProvider(co.cask.cdap.security.impersonation.UnsupportedUGIProvider) DataSetsModules(co.cask.cdap.data.runtime.DataSetsModules) CountDownLatch(java.util.concurrent.CountDownLatch) AuthorizationTestModule(co.cask.cdap.security.authorization.AuthorizationTestModule) LoggerContext(ch.qos.logback.classic.LoggerContext) ServiceUnavailableException(co.cask.cdap.common.ServiceUnavailableException) LoggingModules(co.cask.cdap.logging.guice.LoggingModules) AbstractModule(com.google.inject.AbstractModule) DatasetOpExecutorService(co.cask.cdap.data2.datafabric.dataset.service.executor.DatasetOpExecutorService) StatusListener(ch.qos.logback.core.status.StatusListener) Test(org.junit.Test)

Example 18 with DataFabricModules

use of co.cask.cdap.data.runtime.DataFabricModules in project cdap by caskdata.

the class StandaloneMain method createPersistentModules.

private static List<Module> createPersistentModules(CConfiguration cConf, Configuration hConf) {
    cConf.setIfUnset(Constants.CFG_DATA_LEVELDB_DIR, Constants.DEFAULT_DATA_LEVELDB_DIR);
    cConf.set(Constants.CFG_DATA_INMEMORY_PERSISTENCE, Constants.InMemoryPersistenceType.LEVELDB.name());
    // configure all services except for router and auth to bind to 127.0.0.1
    String localhost = InetAddress.getLoopbackAddress().getHostAddress();
    cConf.set(Constants.Service.MASTER_SERVICES_BIND_ADDRESS, localhost);
    cConf.set(Constants.Transaction.Container.ADDRESS, localhost);
    cConf.set(Constants.Dataset.Executor.ADDRESS, localhost);
    cConf.set(Constants.Stream.ADDRESS, localhost);
    cConf.set(Constants.Metrics.ADDRESS, localhost);
    cConf.set(Constants.Metrics.SERVER_ADDRESS, localhost);
    cConf.set(Constants.MetricsProcessor.ADDRESS, localhost);
    cConf.set(Constants.LogSaver.ADDRESS, localhost);
    cConf.set(Constants.Explore.SERVER_ADDRESS, localhost);
    cConf.set(Constants.Metadata.SERVICE_BIND_ADDRESS, localhost);
    cConf.set(Constants.Preview.ADDRESS, localhost);
    return ImmutableList.of(new ConfigModule(cConf, hConf), new IOModule(), new ZKClientModule(), new KafkaClientModule(), new MetricsHandlerModule(), new DiscoveryRuntimeModule().getStandaloneModules(), new LocationRuntimeModule().getStandaloneModules(), new ProgramRunnerRuntimeModule().getStandaloneModules(), new DataFabricModules(StandaloneMain.class.getName()).getStandaloneModules(), new DataSetsModules().getStandaloneModules(), new DataSetServiceModules().getStandaloneModules(), new MetricsClientRuntimeModule().getStandaloneModules(), new LoggingModules().getStandaloneModules(), new LogReaderRuntimeModules().getStandaloneModules(), new RouterModules().getStandaloneModules(), new SecurityModules().getStandaloneModules(), new SecureStoreModules().getStandaloneModules(), new StreamServiceRuntimeModule().getStandaloneModules(), new ExploreRuntimeModule().getStandaloneModules(), new ServiceStoreModules().getStandaloneModules(), new ExploreClientModule(), new NotificationFeedServiceRuntimeModule().getStandaloneModules(), new NotificationServiceRuntimeModule().getStandaloneModules(), new ViewAdminModules().getStandaloneModules(), new StreamAdminModules().getStandaloneModules(), new NamespaceStoreModule().getStandaloneModules(), new MetadataServiceModule(), new RemoteSystemOperationsServiceModule(), new AuditModule().getStandaloneModules(), new AuthorizationModule(), new AuthorizationEnforcementModule().getStandaloneModules(), new PreviewHttpModule(), new MessagingServerRuntimeModule().getStandaloneModules(), new AppFabricServiceRuntimeModule().getStandaloneModules(), new OperationalStatsModule());
}
Also used : IOModule(co.cask.cdap.common.guice.IOModule) DataSetServiceModules(co.cask.cdap.data.runtime.DataSetServiceModules) ConfigModule(co.cask.cdap.common.guice.ConfigModule) NamespaceStoreModule(co.cask.cdap.store.guice.NamespaceStoreModule) PreviewHttpModule(co.cask.cdap.app.preview.PreviewHttpModule) MessagingServerRuntimeModule(co.cask.cdap.messaging.guice.MessagingServerRuntimeModule) MetricsClientRuntimeModule(co.cask.cdap.metrics.guice.MetricsClientRuntimeModule) NotificationServiceRuntimeModule(co.cask.cdap.notifications.guice.NotificationServiceRuntimeModule) SecurityModules(co.cask.cdap.security.guice.SecurityModules) RemoteSystemOperationsServiceModule(co.cask.cdap.gateway.handlers.meta.RemoteSystemOperationsServiceModule) ViewAdminModules(co.cask.cdap.data.view.ViewAdminModules) ZKClientModule(co.cask.cdap.common.guice.ZKClientModule) KafkaClientModule(co.cask.cdap.common.guice.KafkaClientModule) StreamServiceRuntimeModule(co.cask.cdap.data.stream.service.StreamServiceRuntimeModule) DiscoveryRuntimeModule(co.cask.cdap.common.guice.DiscoveryRuntimeModule) AuthorizationModule(co.cask.cdap.app.guice.AuthorizationModule) OperationalStatsModule(co.cask.cdap.operations.guice.OperationalStatsModule) LogReaderRuntimeModules(co.cask.cdap.logging.guice.LogReaderRuntimeModules) DataSetsModules(co.cask.cdap.data.runtime.DataSetsModules) SecureStoreModules(co.cask.cdap.security.guice.SecureStoreModules) LocationRuntimeModule(co.cask.cdap.common.guice.LocationRuntimeModule) MetadataServiceModule(co.cask.cdap.metadata.MetadataServiceModule) ExploreRuntimeModule(co.cask.cdap.explore.guice.ExploreRuntimeModule) RouterModules(co.cask.cdap.gateway.router.RouterModules) LoggingModules(co.cask.cdap.logging.guice.LoggingModules) StreamAdminModules(co.cask.cdap.data.stream.StreamAdminModules) MetricsHandlerModule(co.cask.cdap.metrics.guice.MetricsHandlerModule) ProgramRunnerRuntimeModule(co.cask.cdap.app.guice.ProgramRunnerRuntimeModule) ExploreClientModule(co.cask.cdap.explore.guice.ExploreClientModule) NotificationFeedServiceRuntimeModule(co.cask.cdap.notifications.feeds.guice.NotificationFeedServiceRuntimeModule) AuditModule(co.cask.cdap.data2.audit.AuditModule) DataFabricModules(co.cask.cdap.data.runtime.DataFabricModules) ServiceStoreModules(co.cask.cdap.app.guice.ServiceStoreModules) AuthorizationEnforcementModule(co.cask.cdap.security.authorization.AuthorizationEnforcementModule) AppFabricServiceRuntimeModule(co.cask.cdap.app.guice.AppFabricServiceRuntimeModule)

Example 19 with DataFabricModules

use of co.cask.cdap.data.runtime.DataFabricModules in project cdap by caskdata.

the class HBaseConsumerStateTest method init.

@BeforeClass
public static void init() throws Exception {
    zkServer = InMemoryZKServer.builder().setDataDir(TMP_FOLDER.newFolder()).build();
    zkServer.startAndWait();
    Configuration hConf = TEST_HBASE.getConfiguration();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TMP_FOLDER.newFolder().getAbsolutePath());
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), new ZKClientModule(), new NonCustomLocationUnitTestModule().getModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new TransactionMetricsModule(), new DataFabricModules().getDistributedModules(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class);
        }
    }, new DataSetsModules().getInMemoryModules(), new SystemDatasetRuntimeModule().getInMemoryModules(), new ExploreClientModule(), new ViewAdminModules().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), Modules.override(new StreamAdminModules().getDistributedModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(StreamMetaStore.class).to(InMemoryStreamMetaStore.class);
            bind(NotificationFeedManager.class).to(NoOpNotificationFeedManager.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    }));
    zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();
    streamAdmin = injector.getInstance(StreamAdmin.class);
    stateStoreFactory = injector.getInstance(StreamConsumerStateStoreFactory.class);
    tableUtil = injector.getInstance(HBaseTableUtil.class);
    ddlExecutor = new HBaseDDLExecutorFactory(cConf, TEST_HBASE.getHBaseAdmin().getConfiguration()).get();
    ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(TEST_NAMESPACE));
    ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(OTHER_NAMESPACE));
    setupNamespaces(injector.getInstance(NamespacedLocationFactory.class));
}
Also used : Configuration(org.apache.hadoop.conf.Configuration) ConfigModule(co.cask.cdap.common.guice.ConfigModule) UGIProvider(co.cask.cdap.security.impersonation.UGIProvider) UnsupportedUGIProvider(co.cask.cdap.security.impersonation.UnsupportedUGIProvider) NamespacedLocationFactory(co.cask.cdap.common.namespace.NamespacedLocationFactory) TransactionMetricsModule(co.cask.cdap.data.runtime.TransactionMetricsModule) ViewAdminModules(co.cask.cdap.data.view.ViewAdminModules) ZKClientModule(co.cask.cdap.common.guice.ZKClientModule) Injector(com.google.inject.Injector) StreamMetaStore(co.cask.cdap.data.stream.service.StreamMetaStore) InMemoryStreamMetaStore(co.cask.cdap.data.stream.service.InMemoryStreamMetaStore) SimpleNamespaceQueryAdmin(co.cask.cdap.common.namespace.SimpleNamespaceQueryAdmin) HBaseDDLExecutorFactory(co.cask.cdap.data2.util.hbase.HBaseDDLExecutorFactory) SystemDatasetRuntimeModule(co.cask.cdap.data.runtime.SystemDatasetRuntimeModule) DiscoveryRuntimeModule(co.cask.cdap.common.guice.DiscoveryRuntimeModule) NotificationFeedManager(co.cask.cdap.notifications.feeds.NotificationFeedManager) NoOpNotificationFeedManager(co.cask.cdap.notifications.feeds.service.NoOpNotificationFeedManager) AuthenticationContextModules(co.cask.cdap.security.auth.context.AuthenticationContextModules) DataSetsModules(co.cask.cdap.data.runtime.DataSetsModules) DefaultOwnerAdmin(co.cask.cdap.security.impersonation.DefaultOwnerAdmin) OwnerAdmin(co.cask.cdap.security.impersonation.OwnerAdmin) NonCustomLocationUnitTestModule(co.cask.cdap.common.guice.NonCustomLocationUnitTestModule) AuthorizationTestModule(co.cask.cdap.security.authorization.AuthorizationTestModule) HBaseTableUtil(co.cask.cdap.data2.util.hbase.HBaseTableUtil) AbstractModule(com.google.inject.AbstractModule) StreamAdminModules(co.cask.cdap.data.stream.StreamAdminModules) StreamAdmin(co.cask.cdap.data2.transaction.stream.StreamAdmin) ZKClientService(org.apache.twill.zookeeper.ZKClientService) ExploreClientModule(co.cask.cdap.explore.guice.ExploreClientModule) DataFabricModules(co.cask.cdap.data.runtime.DataFabricModules) AuthorizationEnforcementModule(co.cask.cdap.security.authorization.AuthorizationEnforcementModule) StreamConsumerStateStoreFactory(co.cask.cdap.data2.transaction.stream.StreamConsumerStateStoreFactory) BeforeClass(org.junit.BeforeClass)

Example 20 with DataFabricModules

use of co.cask.cdap.data.runtime.DataFabricModules in project cdap by caskdata.

the class HBaseStreamConsumerTest method init.

@BeforeClass
public static void init() throws Exception {
    zkServer = InMemoryZKServer.builder().setDataDir(TMP_FOLDER.newFolder()).build();
    zkServer.startAndWait();
    Configuration hConf = TEST_HBASE.getConfiguration();
    cConf.setInt(Constants.Stream.CONTAINER_INSTANCES, 1);
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TMP_FOLDER.newFolder().getAbsolutePath());
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), new ZKClientModule(), new NonCustomLocationUnitTestModule().getModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new TransactionMetricsModule(), new DataSetsModules().getInMemoryModules(), new SystemDatasetRuntimeModule().getInMemoryModules(), new ExploreClientModule(), new ViewAdminModules().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), Modules.override(new DataFabricModules().getDistributedModules(), new StreamAdminModules().getDistributedModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(TransactionStateStorage.class).to(NoOpTransactionStateStorage.class);
            bind(TransactionSystemClient.class).to(InMemoryTxSystemClient.class).in(Singleton.class);
            bind(StreamMetaStore.class).to(InMemoryStreamMetaStore.class);
            bind(NotificationFeedManager.class).to(NoOpNotificationFeedManager.class);
            bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    }));
    zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();
    streamAdmin = injector.getInstance(StreamAdmin.class);
    consumerFactory = injector.getInstance(StreamConsumerFactory.class);
    txClient = injector.getInstance(TransactionSystemClient.class);
    txManager = TxInMemory.getTransactionManager(txClient);
    queueClientFactory = injector.getInstance(QueueClientFactory.class);
    fileWriterFactory = injector.getInstance(StreamFileWriterFactory.class);
    txManager.startAndWait();
    tableUtil = injector.getInstance(HBaseTableUtil.class);
    ddlExecutor = new HBaseDDLExecutorFactory(cConf, TEST_HBASE.getHBaseAdmin().getConfiguration()).get();
    ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(NamespaceId.SYSTEM));
    ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(TEST_NAMESPACE));
    ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(OTHER_NAMESPACE));
    setupNamespaces(injector.getInstance(NamespacedLocationFactory.class));
}
Also used : Configuration(org.apache.hadoop.conf.Configuration) ConfigModule(co.cask.cdap.common.guice.ConfigModule) UGIProvider(co.cask.cdap.security.impersonation.UGIProvider) UnsupportedUGIProvider(co.cask.cdap.security.impersonation.UnsupportedUGIProvider) NamespacedLocationFactory(co.cask.cdap.common.namespace.NamespacedLocationFactory) StreamConsumerFactory(co.cask.cdap.data2.transaction.stream.StreamConsumerFactory) TransactionMetricsModule(co.cask.cdap.data.runtime.TransactionMetricsModule) ViewAdminModules(co.cask.cdap.data.view.ViewAdminModules) ZKClientModule(co.cask.cdap.common.guice.ZKClientModule) TransactionSystemClient(org.apache.tephra.TransactionSystemClient) Injector(com.google.inject.Injector) StreamMetaStore(co.cask.cdap.data.stream.service.StreamMetaStore) InMemoryStreamMetaStore(co.cask.cdap.data.stream.service.InMemoryStreamMetaStore) SimpleNamespaceQueryAdmin(co.cask.cdap.common.namespace.SimpleNamespaceQueryAdmin) NamespaceQueryAdmin(co.cask.cdap.common.namespace.NamespaceQueryAdmin) HBaseDDLExecutorFactory(co.cask.cdap.data2.util.hbase.HBaseDDLExecutorFactory) SystemDatasetRuntimeModule(co.cask.cdap.data.runtime.SystemDatasetRuntimeModule) DiscoveryRuntimeModule(co.cask.cdap.common.guice.DiscoveryRuntimeModule) NotificationFeedManager(co.cask.cdap.notifications.feeds.NotificationFeedManager) NoOpNotificationFeedManager(co.cask.cdap.notifications.feeds.service.NoOpNotificationFeedManager) AuthenticationContextModules(co.cask.cdap.security.auth.context.AuthenticationContextModules) DataSetsModules(co.cask.cdap.data.runtime.DataSetsModules) DefaultOwnerAdmin(co.cask.cdap.security.impersonation.DefaultOwnerAdmin) OwnerAdmin(co.cask.cdap.security.impersonation.OwnerAdmin) NonCustomLocationUnitTestModule(co.cask.cdap.common.guice.NonCustomLocationUnitTestModule) InMemoryTxSystemClient(org.apache.tephra.inmemory.InMemoryTxSystemClient) AuthorizationTestModule(co.cask.cdap.security.authorization.AuthorizationTestModule) HBaseTableUtil(co.cask.cdap.data2.util.hbase.HBaseTableUtil) AbstractModule(com.google.inject.AbstractModule) StreamAdminModules(co.cask.cdap.data.stream.StreamAdminModules) StreamAdmin(co.cask.cdap.data2.transaction.stream.StreamAdmin) StreamFileWriterFactory(co.cask.cdap.data.stream.StreamFileWriterFactory) ZKClientService(org.apache.twill.zookeeper.ZKClientService) ExploreClientModule(co.cask.cdap.explore.guice.ExploreClientModule) QueueClientFactory(co.cask.cdap.data2.queue.QueueClientFactory) NoOpTransactionStateStorage(org.apache.tephra.persist.NoOpTransactionStateStorage) TransactionStateStorage(org.apache.tephra.persist.TransactionStateStorage) DataFabricModules(co.cask.cdap.data.runtime.DataFabricModules) AuthorizationEnforcementModule(co.cask.cdap.security.authorization.AuthorizationEnforcementModule) BeforeClass(org.junit.BeforeClass)

Aggregations

DataFabricModules (co.cask.cdap.data.runtime.DataFabricModules)34 ConfigModule (co.cask.cdap.common.guice.ConfigModule)33 DataSetsModules (co.cask.cdap.data.runtime.DataSetsModules)33 AuthorizationEnforcementModule (co.cask.cdap.security.authorization.AuthorizationEnforcementModule)33 DiscoveryRuntimeModule (co.cask.cdap.common.guice.DiscoveryRuntimeModule)32 AbstractModule (com.google.inject.AbstractModule)30 AuthenticationContextModules (co.cask.cdap.security.auth.context.AuthenticationContextModules)28 ExploreClientModule (co.cask.cdap.explore.guice.ExploreClientModule)27 AuthorizationTestModule (co.cask.cdap.security.authorization.AuthorizationTestModule)26 UnsupportedUGIProvider (co.cask.cdap.security.impersonation.UnsupportedUGIProvider)25 NonCustomLocationUnitTestModule (co.cask.cdap.common.guice.NonCustomLocationUnitTestModule)23 ViewAdminModules (co.cask.cdap.data.view.ViewAdminModules)20 DefaultOwnerAdmin (co.cask.cdap.security.impersonation.DefaultOwnerAdmin)20 Injector (com.google.inject.Injector)19 CConfiguration (co.cask.cdap.common.conf.CConfiguration)17 ZKClientModule (co.cask.cdap.common.guice.ZKClientModule)17 BeforeClass (org.junit.BeforeClass)17 DataSetServiceModules (co.cask.cdap.data.runtime.DataSetServiceModules)16 TransactionMetricsModule (co.cask.cdap.data.runtime.TransactionMetricsModule)16 StreamAdminModules (co.cask.cdap.data.stream.StreamAdminModules)16