Examples with DataFabricModules io.cdap.cdap.data.runtime.DataFabricModules used on opensource projects

Example 31 with DataFabricModules

use of io.cdap.cdap.data.runtime.DataFabricModules in project cdap by caskdata.

the class BaseHiveExploreServiceTest method createInMemoryModules.

private static List<Module> createInMemoryModules(CConfiguration configuration, Configuration hConf, TemporaryFolder tmpFolder) throws IOException {
    configuration.set(Constants.CFG_DATA_INMEMORY_PERSISTENCE, Constants.InMemoryPersistenceType.MEMORY.name());
    configuration.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath());
    configuration.set(Constants.Explore.LOCAL_DATA_DIR, tmpFolder.newFolder("hive").getAbsolutePath());
    configuration.set(TxConstants.Manager.CFG_TX_SNAPSHOT_LOCAL_DIR, tmpFolder.newFolder("tx").getAbsolutePath());
    configuration.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, true);
    return ImmutableList.of(new ConfigModule(configuration, hConf), RemoteAuthenticatorModules.getNoOpModule(), new IOModule(), new InMemoryDiscoveryModule(), new MessagingServerRuntimeModule().getInMemoryModules(), new NonCustomLocationUnitTestModule(), new DataSetsModules().getStandaloneModules(), new DataSetServiceModules().getInMemoryModules(), new MetricsClientRuntimeModule().getInMemoryModules(), new ExploreRuntimeModule().getInMemoryModules(), new ExploreClientModule(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getMasterModule(), new NamespaceAdminTestModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
            bind(MetadataServiceClient.class).to(NoOpMetadataServiceClient.class);
            // Use LocalFileTransactionStateStorage, so that we can use transaction snapshots for assertions in test
            install(Modules.override(new DataFabricModules().getInMemoryModules()).with(new AbstractModule() {

                @Override
                protected void configure() {
                    bind(TransactionStateStorage.class).annotatedWith(Names.named("persist")).to(LocalFileTransactionStateStorage.class).in(Scopes.SINGLETON);
                    bind(TransactionStateStorage.class).toProvider(TransactionStateStorageProvider.class).in(Singleton.class);
                }
            }));
        }
    });
}

Example 32 with DataFabricModules

use of io.cdap.cdap.data.runtime.DataFabricModules in project cdap by caskdata.

the class StandaloneMain method createPersistentModules.

private static List<Module> createPersistentModules(CConfiguration cConf, Configuration hConf) {
    cConf.setInt(Constants.Master.MAX_INSTANCES, 1);
    cConf.setIfUnset(Constants.CFG_DATA_LEVELDB_DIR, Constants.DEFAULT_DATA_LEVELDB_DIR);
    cConf.set(Constants.CFG_DATA_INMEMORY_PERSISTENCE, Constants.InMemoryPersistenceType.LEVELDB.name());
    // configure all services except for router and auth to bind to 127.0.0.1
    String localhost = InetAddress.getLoopbackAddress().getHostAddress();
    cConf.set(Constants.Service.MASTER_SERVICES_BIND_ADDRESS, localhost);
    cConf.set(Constants.MessagingSystem.HTTP_SERVER_BIND_ADDRESS, localhost);
    cConf.set(Constants.Transaction.Container.ADDRESS, localhost);
    cConf.set(Constants.Dataset.Executor.ADDRESS, localhost);
    cConf.set(Constants.Metrics.ADDRESS, localhost);
    cConf.set(Constants.MetricsProcessor.BIND_ADDRESS, localhost);
    cConf.set(Constants.LogSaver.ADDRESS, localhost);
    cConf.set(Constants.LogQuery.ADDRESS, localhost);
    cConf.set(Constants.Explore.SERVER_ADDRESS, localhost);
    cConf.set(Constants.Metadata.SERVICE_BIND_ADDRESS, localhost);
    cConf.set(Constants.Preview.ADDRESS, localhost);
    cConf.set(Constants.SupportBundle.SERVICE_BIND_ADDRESS, localhost);
    return ImmutableList.of(new ConfigModule(cConf, hConf), RemoteAuthenticatorModules.getDefaultModule(), new IOModule(), new ZKClientModule(), new KafkaClientModule(), new MetricsHandlerModule(), new LogQueryRuntimeModule().getStandaloneModules(), new InMemoryDiscoveryModule(), new LocalLocationModule(), new ProgramRunnerRuntimeModule().getStandaloneModules(), new DataFabricModules(StandaloneMain.class.getName()).getStandaloneModules(), new DataSetsModules().getStandaloneModules(), new DataSetServiceModules().getStandaloneModules(), new MetricsClientRuntimeModule().getStandaloneModules(), new LocalLogAppenderModule(), new LogReaderRuntimeModules().getStandaloneModules(), new RouterModules().getStandaloneModules(), new CoreSecurityRuntimeModule().getStandaloneModules(), new ExternalAuthenticationModule(), new SecureStoreServerModule(), new ExploreRuntimeModule().getStandaloneModules(), new ExploreClientModule(), new MetadataServiceModule(), new MetadataReaderWriterModules().getStandaloneModules(), new AuditModule(), new AuthenticationContextModules().getMasterModule(), new AuthorizationModule(), new AuthorizationEnforcementModule().getStandaloneModules(), new PreviewConfigModule(cConf, new Configuration(), SConfiguration.create()), new PreviewManagerModule(false), new PreviewRunnerManagerModule().getStandaloneModules(), new MessagingServerRuntimeModule().getStandaloneModules(), new AppFabricServiceRuntimeModule(cConf).getStandaloneModules(), new MonitorHandlerModule(false), new RuntimeServerModule(), new OperationalStatsModule(), new MetricsWriterModule(), new SupportBundleServiceModule(), new AbstractModule() {

        @Override
        protected void configure() {
            // Needed by MonitorHandlerModuler
            bind(TwillRunner.class).to(NoopTwillRunnerService.class);
            bind(HealthCheckService.class).in(Scopes.SINGLETON);
        }
    });
}

Example 33 with DataFabricModules

use of io.cdap.cdap.data.runtime.DataFabricModules in project cdap by caskdata.

the class DatasetBasedTimeScheduleStoreTest method beforeClass.

@BeforeClass
public static void beforeClass() throws Exception {
    CConfiguration conf = CConfiguration.create();
    conf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder("data").getAbsolutePath());
    injector = Guice.createInjector(new ConfigModule(conf), RemoteAuthenticatorModules.getNoOpModule(), new NonCustomLocationUnitTestModule(), new InMemoryDiscoveryModule(), new MetricsClientRuntimeModule().getInMemoryModules(), new DataFabricModules().getInMemoryModules(), new DataSetsModules().getStandaloneModules(), new DataSetServiceModules().getInMemoryModules(), new ExploreClientModule(), new NamespaceAdminTestModule(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getMasterModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(NoOpOwnerAdmin.class);
            bind(MetadataServiceClient.class).to(NoOpMetadataServiceClient.class);
        }
    });
    txService = injector.getInstance(TransactionManager.class);
    txService.startAndWait();
    StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class));
    dsOpsService = injector.getInstance(DatasetOpExecutorService.class);
    dsOpsService.startAndWait();
    dsService = injector.getInstance(DatasetService.class);
    dsService.startAndWait();
    transactionRunner = injector.getInstance(TransactionRunner.class);
}

Example 34 with DataFabricModules

use of io.cdap.cdap.data.runtime.DataFabricModules in project cdap by cdapio.

the class PreviewRunnerTwillRunnable method createInjector.

@VisibleForTesting
static Injector createInjector(CConfiguration cConf, Configuration hConf, PreviewRequestPollerInfo pollerInfo) {
    List<Module> modules = new ArrayList<>();
    byte[] pollerInfoBytes = Bytes.toBytes(new Gson().toJson(pollerInfo));
    SConfiguration sConf = SConfiguration.create();
    modules.add(new ConfigModule(cConf, hConf, sConf));
    modules.add(RemoteAuthenticatorModules.getDefaultModule());
    modules.add(new PreviewConfigModule(cConf, hConf, sConf));
    modules.add(new IOModule());
    modules.add(new MetricsClientRuntimeModule().getDistributedModules());
    // If MasterEnvironment is not available, assuming it is the old hadoop stack with ZK, Kafka
    MasterEnvironment masterEnv = MasterEnvironments.getMasterEnvironment();
    if (masterEnv == null) {
        modules.add(new ZKClientModule());
        modules.add(new ZKDiscoveryModule());
        modules.add(new KafkaClientModule());
        modules.add(new KafkaLogAppenderModule());
    } else {
        modules.add(new AbstractModule() {

            @Override
            protected void configure() {
                bind(DiscoveryService.class).toProvider(new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceSupplier()));
                bind(DiscoveryServiceClient.class).toProvider(new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceClientSupplier()));
            }
        });
        modules.add(new RemoteLogAppenderModule());
    }
    modules.add(new PreviewRunnerManagerModule().getDistributedModules());
    modules.add(new DataSetServiceModules().getStandaloneModules());
    modules.add(new DataSetsModules().getStandaloneModules());
    modules.add(new AppFabricServiceRuntimeModule(cConf).getStandaloneModules());
    modules.add(new ProgramRunnerRuntimeModule().getStandaloneModules());
    modules.add(new MetricsStoreModule());
    modules.add(new MessagingClientModule());
    modules.add(new AuditModule());
    modules.add(new SecureStoreClientModule());
    modules.add(new MetadataReaderWriterModules().getStandaloneModules());
    modules.add(new DFSLocationModule());
    modules.add(new MetadataServiceModule());
    modules.add(new CoreSecurityRuntimeModule().getInMemoryModules());
    modules.add(new AuthenticationContextModules().getMasterWorkerModule());
    modules.add(new AuthorizationModule());
    modules.add(new AuthorizationEnforcementModule().getNoOpModules());
    modules.add(Modules.override(new DataFabricModules("master").getDistributedModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            // Bind transaction system to a constant one, basically no transaction, with every write become
            // visible immediately.
            // TODO: Ideally we shouldn't need this at all. However, it is needed now to satisfy dependencies
            bind(TransactionSystemClientService.class).to(DelegatingTransactionSystemClientService.class);
            bind(TransactionSystemClient.class).to(ConstantTransactionSystemClient.class);
            bind(ExploreClient.class).to(UnsupportedExploreClient.class);
            bind(PreviewRequestPollerInfoProvider.class).toInstance(() -> pollerInfoBytes);
        }
    }));
    return Guice.createInjector(modules);
}

Example 35 with DataFabricModules

use of io.cdap.cdap.data.runtime.DataFabricModules in project cdap by cdapio.

the class LogHttpHandlerTest method setup.

@BeforeClass
public static void setup() throws Exception {
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
    cConf.set(Constants.LogQuery.ADDRESS, InetAddress.getLoopbackAddress().getHostAddress());
    Injector injector = Guice.createInjector(Modules.override(new ConfigModule(cConf), RemoteAuthenticatorModules.getNoOpModule(), new NonCustomLocationUnitTestModule(), new InMemoryDiscoveryModule(), new LogQueryRuntimeModule().getInMemoryModules(), new DataFabricModules().getInMemoryModules(), new DataSetsModules().getStandaloneModules(), new DataSetServiceModules().getInMemoryModules(), new ExploreClientModule(), new NamespaceAdminTestModule(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getMasterModule()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class);
            bind(LogReader.class).to(MockLogReader.class).in(Scopes.SINGLETON);
            bind(Store.class).to(DefaultStore.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(NoOpOwnerAdmin.class);
            // TODO (CDAP-14677): find a better way to inject metadata publisher
            bind(MetadataServiceClient.class).to(NoOpMetadataServiceClient.class);
        }
    }));
    transactionManager = injector.getInstance(TransactionManager.class);
    transactionManager.startAndWait();
    StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class));
    dsOpService = injector.getInstance(DatasetOpExecutorService.class);
    dsOpService.startAndWait();
    datasetService = injector.getInstance(DatasetService.class);
    datasetService.startAndWait();
    logQueryService = injector.getInstance(LogQueryService.class);
    logQueryService.startAndWait();
    mockLogReader = (MockLogReader) injector.getInstance(LogReader.class);
    mockLogReader.generateLogs();
    discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class);
}